rustc_codegen_llvm/
builder.rs

1use std::borrow::{Borrow, Cow};
2use std::ops::Deref;
3use std::{iter, ptr};
4
5pub(crate) mod autodiff;
6
7use libc::{c_char, c_uint, size_t};
8use rustc_abi as abi;
9use rustc_abi::{Align, Size, WrappingRange};
10use rustc_codegen_ssa::MemFlags;
11use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind};
12use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
13use rustc_codegen_ssa::mir::place::PlaceRef;
14use rustc_codegen_ssa::traits::*;
15use rustc_data_structures::small_c_str::SmallCStr;
16use rustc_hir::def_id::DefId;
17use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
18use rustc_middle::ty::layout::{
19    FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers,
20    TyAndLayout,
21};
22use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
23use rustc_sanitizers::{cfi, kcfi};
24use rustc_session::config::OptLevel;
25use rustc_span::Span;
26use rustc_target::callconv::FnAbi;
27use rustc_target::spec::{HasTargetSpec, SanitizerSet, Target};
28use smallvec::SmallVec;
29use tracing::{debug, instrument};
30
31use crate::abi::FnAbiLlvmExt;
32use crate::attributes;
33use crate::common::Funclet;
34use crate::context::{CodegenCx, FullCx, GenericCx, SCx};
35use crate::llvm::{
36    self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, GEPNoWrapFlags, Metadata, True,
37};
38use crate::type_::Type;
39use crate::type_of::LayoutLlvmExt;
40use crate::value::Value;
41
42#[must_use]
43pub(crate) struct GenericBuilder<'a, 'll, CX: Borrow<SCx<'ll>>> {
44    pub llbuilder: &'ll mut llvm::Builder<'ll>,
45    pub cx: &'a GenericCx<'ll, CX>,
46}
47
48pub(crate) type SBuilder<'a, 'll> = GenericBuilder<'a, 'll, SCx<'ll>>;
49pub(crate) type Builder<'a, 'll, 'tcx> = GenericBuilder<'a, 'll, FullCx<'ll, 'tcx>>;
50
51impl<'a, 'll, CX: Borrow<SCx<'ll>>> Drop for GenericBuilder<'a, 'll, CX> {
52    fn drop(&mut self) {
53        unsafe {
54            llvm::LLVMDisposeBuilder(&mut *(self.llbuilder as *mut _));
55        }
56    }
57}
58
59impl<'a, 'll> SBuilder<'a, 'll> {
60    pub(crate) fn call(
61        &mut self,
62        llty: &'ll Type,
63        llfn: &'ll Value,
64        args: &[&'ll Value],
65        funclet: Option<&Funclet<'ll>>,
66    ) -> &'ll Value {
67        debug!("call {:?} with args ({:?})", llfn, args);
68
69        let args = self.check_call("call", llty, llfn, args);
70        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
71        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
72        if let Some(funclet_bundle) = funclet_bundle {
73            bundles.push(funclet_bundle);
74        }
75
76        let call = unsafe {
77            llvm::LLVMBuildCallWithOperandBundles(
78                self.llbuilder,
79                llty,
80                llfn,
81                args.as_ptr() as *const &llvm::Value,
82                args.len() as c_uint,
83                bundles.as_ptr(),
84                bundles.len() as c_uint,
85                c"".as_ptr(),
86            )
87        };
88        call
89    }
90}
91
92impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
93    fn with_cx(scx: &'a GenericCx<'ll, CX>) -> Self {
94        // Create a fresh builder from the simple context.
95        let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(scx.deref().borrow().llcx) };
96        GenericBuilder { llbuilder, cx: scx }
97    }
98
99    pub(crate) fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
100        unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) }
101    }
102
103    pub(crate) fn ret_void(&mut self) {
104        llvm::LLVMBuildRetVoid(self.llbuilder);
105    }
106
107    pub(crate) fn ret(&mut self, v: &'ll Value) {
108        unsafe {
109            llvm::LLVMBuildRet(self.llbuilder, v);
110        }
111    }
112
113    pub(crate) fn build(cx: &'a GenericCx<'ll, CX>, llbb: &'ll BasicBlock) -> Self {
114        let bx = Self::with_cx(cx);
115        unsafe {
116            llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb);
117        }
118        bx
119    }
120}
121
122/// Empty string, to be used where LLVM expects an instruction name, indicating
123/// that the instruction is to be left unnamed (i.e. numbered, in textual IR).
124// FIXME(eddyb) pass `&CStr` directly to FFI once it's a thin pointer.
125pub(crate) const UNNAMED: *const c_char = c"".as_ptr();
126
127impl<'ll, CX: Borrow<SCx<'ll>>> BackendTypes for GenericBuilder<'_, 'll, CX> {
128    type Value = <GenericCx<'ll, CX> as BackendTypes>::Value;
129    type Metadata = <GenericCx<'ll, CX> as BackendTypes>::Metadata;
130    type Function = <GenericCx<'ll, CX> as BackendTypes>::Function;
131    type BasicBlock = <GenericCx<'ll, CX> as BackendTypes>::BasicBlock;
132    type Type = <GenericCx<'ll, CX> as BackendTypes>::Type;
133    type Funclet = <GenericCx<'ll, CX> as BackendTypes>::Funclet;
134
135    type DIScope = <GenericCx<'ll, CX> as BackendTypes>::DIScope;
136    type DILocation = <GenericCx<'ll, CX> as BackendTypes>::DILocation;
137    type DIVariable = <GenericCx<'ll, CX> as BackendTypes>::DIVariable;
138}
139
140impl abi::HasDataLayout for Builder<'_, '_, '_> {
141    fn data_layout(&self) -> &abi::TargetDataLayout {
142        self.cx.data_layout()
143    }
144}
145
146impl<'tcx> ty::layout::HasTyCtxt<'tcx> for Builder<'_, '_, 'tcx> {
147    #[inline]
148    fn tcx(&self) -> TyCtxt<'tcx> {
149        self.cx.tcx
150    }
151}
152
153impl<'tcx> ty::layout::HasTypingEnv<'tcx> for Builder<'_, '_, 'tcx> {
154    fn typing_env(&self) -> ty::TypingEnv<'tcx> {
155        self.cx.typing_env()
156    }
157}
158
159impl HasTargetSpec for Builder<'_, '_, '_> {
160    #[inline]
161    fn target_spec(&self) -> &Target {
162        self.cx.target_spec()
163    }
164}
165
166impl<'tcx> LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
167    #[inline]
168    fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
169        self.cx.handle_layout_err(err, span, ty)
170    }
171}
172
173impl<'tcx> FnAbiOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
174    #[inline]
175    fn handle_fn_abi_err(
176        &self,
177        err: FnAbiError<'tcx>,
178        span: Span,
179        fn_abi_request: FnAbiRequest<'tcx>,
180    ) -> ! {
181        self.cx.handle_fn_abi_err(err, span, fn_abi_request)
182    }
183}
184
185impl<'ll, 'tcx> Deref for Builder<'_, 'll, 'tcx> {
186    type Target = CodegenCx<'ll, 'tcx>;
187
188    #[inline]
189    fn deref(&self) -> &Self::Target {
190        self.cx
191    }
192}
193
194macro_rules! math_builder_methods {
195    ($($name:ident($($arg:ident),*) => $llvm_capi:ident),+ $(,)?) => {
196        $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
197            unsafe {
198                llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED)
199            }
200        })+
201    }
202}
203
204macro_rules! set_math_builder_methods {
205    ($($name:ident($($arg:ident),*) => ($llvm_capi:ident, $llvm_set_math:ident)),+ $(,)?) => {
206        $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
207            unsafe {
208                let instr = llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED);
209                llvm::$llvm_set_math(instr);
210                instr
211            }
212        })+
213    }
214}
215
216impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
217    type CodegenCx = CodegenCx<'ll, 'tcx>;
218
219    fn build(cx: &'a CodegenCx<'ll, 'tcx>, llbb: &'ll BasicBlock) -> Self {
220        let bx = Builder::with_cx(cx);
221        unsafe {
222            llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb);
223        }
224        bx
225    }
226
227    fn cx(&self) -> &CodegenCx<'ll, 'tcx> {
228        self.cx
229    }
230
231    fn llbb(&self) -> &'ll BasicBlock {
232        unsafe { llvm::LLVMGetInsertBlock(self.llbuilder) }
233    }
234
235    fn set_span(&mut self, _span: Span) {}
236
237    fn append_block(cx: &'a CodegenCx<'ll, 'tcx>, llfn: &'ll Value, name: &str) -> &'ll BasicBlock {
238        unsafe {
239            let name = SmallCStr::new(name);
240            llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, name.as_ptr())
241        }
242    }
243
244    fn append_sibling_block(&mut self, name: &str) -> &'ll BasicBlock {
245        Self::append_block(self.cx, self.llfn(), name)
246    }
247
248    fn switch_to_block(&mut self, llbb: Self::BasicBlock) {
249        *self = Self::build(self.cx, llbb)
250    }
251
252    fn ret_void(&mut self) {
253        llvm::LLVMBuildRetVoid(self.llbuilder);
254    }
255
256    fn ret(&mut self, v: &'ll Value) {
257        unsafe {
258            llvm::LLVMBuildRet(self.llbuilder, v);
259        }
260    }
261
262    fn br(&mut self, dest: &'ll BasicBlock) {
263        unsafe {
264            llvm::LLVMBuildBr(self.llbuilder, dest);
265        }
266    }
267
268    fn cond_br(
269        &mut self,
270        cond: &'ll Value,
271        then_llbb: &'ll BasicBlock,
272        else_llbb: &'ll BasicBlock,
273    ) {
274        unsafe {
275            llvm::LLVMBuildCondBr(self.llbuilder, cond, then_llbb, else_llbb);
276        }
277    }
278
279    fn switch(
280        &mut self,
281        v: &'ll Value,
282        else_llbb: &'ll BasicBlock,
283        cases: impl ExactSizeIterator<Item = (u128, &'ll BasicBlock)>,
284    ) {
285        let switch =
286            unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) };
287        for (on_val, dest) in cases {
288            let on_val = self.const_uint_big(self.val_ty(v), on_val);
289            unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
290        }
291    }
292
293    fn switch_with_weights(
294        &mut self,
295        v: Self::Value,
296        else_llbb: Self::BasicBlock,
297        else_is_cold: bool,
298        cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock, bool)>,
299    ) {
300        if self.cx.sess().opts.optimize == rustc_session::config::OptLevel::No {
301            self.switch(v, else_llbb, cases.map(|(val, dest, _)| (val, dest)));
302            return;
303        }
304
305        let id_str = "branch_weights";
306        let id = unsafe {
307            llvm::LLVMMDStringInContext2(self.cx.llcx, id_str.as_ptr().cast(), id_str.len())
308        };
309
310        // For switch instructions with 2 targets, the `llvm.expect` intrinsic is used.
311        // This function handles switch instructions with more than 2 targets and it needs to
312        // emit branch weights metadata instead of using the intrinsic.
313        // The values 1 and 2000 are the same as the values used by the `llvm.expect` intrinsic.
314        let cold_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(1));
315        let hot_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(2000));
316        let weight =
317            |is_cold: bool| -> &Metadata { if is_cold { cold_weight } else { hot_weight } };
318
319        let mut md: SmallVec<[&Metadata; 16]> = SmallVec::with_capacity(cases.len() + 2);
320        md.push(id);
321        md.push(weight(else_is_cold));
322
323        let switch =
324            unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) };
325        for (on_val, dest, is_cold) in cases {
326            let on_val = self.const_uint_big(self.val_ty(v), on_val);
327            unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
328            md.push(weight(is_cold));
329        }
330
331        unsafe {
332            let md_node = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len() as size_t);
333            self.cx.set_metadata(switch, llvm::MD_prof, md_node);
334        }
335    }
336
337    fn invoke(
338        &mut self,
339        llty: &'ll Type,
340        fn_attrs: Option<&CodegenFnAttrs>,
341        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
342        llfn: &'ll Value,
343        args: &[&'ll Value],
344        then: &'ll BasicBlock,
345        catch: &'ll BasicBlock,
346        funclet: Option<&Funclet<'ll>>,
347        instance: Option<Instance<'tcx>>,
348    ) -> &'ll Value {
349        debug!("invoke {:?} with args ({:?})", llfn, args);
350
351        let args = self.check_call("invoke", llty, llfn, args);
352        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
353        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
354        if let Some(funclet_bundle) = funclet_bundle {
355            bundles.push(funclet_bundle);
356        }
357
358        // Emit CFI pointer type membership test
359        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
360
361        // Emit KCFI operand bundle
362        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
363        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
364            bundles.push(kcfi_bundle);
365        }
366
367        let invoke = unsafe {
368            llvm::LLVMBuildInvokeWithOperandBundles(
369                self.llbuilder,
370                llty,
371                llfn,
372                args.as_ptr(),
373                args.len() as c_uint,
374                then,
375                catch,
376                bundles.as_ptr(),
377                bundles.len() as c_uint,
378                UNNAMED,
379            )
380        };
381        if let Some(fn_abi) = fn_abi {
382            fn_abi.apply_attrs_callsite(self, invoke);
383        }
384        invoke
385    }
386
387    fn unreachable(&mut self) {
388        unsafe {
389            llvm::LLVMBuildUnreachable(self.llbuilder);
390        }
391    }
392
393    math_builder_methods! {
394        add(a, b) => LLVMBuildAdd,
395        fadd(a, b) => LLVMBuildFAdd,
396        sub(a, b) => LLVMBuildSub,
397        fsub(a, b) => LLVMBuildFSub,
398        mul(a, b) => LLVMBuildMul,
399        fmul(a, b) => LLVMBuildFMul,
400        udiv(a, b) => LLVMBuildUDiv,
401        exactudiv(a, b) => LLVMBuildExactUDiv,
402        sdiv(a, b) => LLVMBuildSDiv,
403        exactsdiv(a, b) => LLVMBuildExactSDiv,
404        fdiv(a, b) => LLVMBuildFDiv,
405        urem(a, b) => LLVMBuildURem,
406        srem(a, b) => LLVMBuildSRem,
407        frem(a, b) => LLVMBuildFRem,
408        shl(a, b) => LLVMBuildShl,
409        lshr(a, b) => LLVMBuildLShr,
410        ashr(a, b) => LLVMBuildAShr,
411        and(a, b) => LLVMBuildAnd,
412        or(a, b) => LLVMBuildOr,
413        xor(a, b) => LLVMBuildXor,
414        neg(x) => LLVMBuildNeg,
415        fneg(x) => LLVMBuildFNeg,
416        not(x) => LLVMBuildNot,
417        unchecked_sadd(x, y) => LLVMBuildNSWAdd,
418        unchecked_uadd(x, y) => LLVMBuildNUWAdd,
419        unchecked_ssub(x, y) => LLVMBuildNSWSub,
420        unchecked_usub(x, y) => LLVMBuildNUWSub,
421        unchecked_smul(x, y) => LLVMBuildNSWMul,
422        unchecked_umul(x, y) => LLVMBuildNUWMul,
423    }
424
425    fn unchecked_suadd(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
426        unsafe {
427            let add = llvm::LLVMBuildAdd(self.llbuilder, a, b, UNNAMED);
428            if llvm::LLVMIsAInstruction(add).is_some() {
429                llvm::LLVMSetNUW(add, True);
430                llvm::LLVMSetNSW(add, True);
431            }
432            add
433        }
434    }
435    fn unchecked_susub(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
436        unsafe {
437            let sub = llvm::LLVMBuildSub(self.llbuilder, a, b, UNNAMED);
438            if llvm::LLVMIsAInstruction(sub).is_some() {
439                llvm::LLVMSetNUW(sub, True);
440                llvm::LLVMSetNSW(sub, True);
441            }
442            sub
443        }
444    }
445    fn unchecked_sumul(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
446        unsafe {
447            let mul = llvm::LLVMBuildMul(self.llbuilder, a, b, UNNAMED);
448            if llvm::LLVMIsAInstruction(mul).is_some() {
449                llvm::LLVMSetNUW(mul, True);
450                llvm::LLVMSetNSW(mul, True);
451            }
452            mul
453        }
454    }
455
456    fn or_disjoint(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
457        unsafe {
458            let or = llvm::LLVMBuildOr(self.llbuilder, a, b, UNNAMED);
459
460            // If a and b are both values, then `or` is a value, rather than
461            // an instruction, so we need to check before setting the flag.
462            // (See also `LLVMBuildNUWNeg` which also needs a check.)
463            if llvm::LLVMIsAInstruction(or).is_some() {
464                llvm::LLVMSetIsDisjoint(or, True);
465            }
466            or
467        }
468    }
469
470    set_math_builder_methods! {
471        fadd_fast(x, y) => (LLVMBuildFAdd, LLVMRustSetFastMath),
472        fsub_fast(x, y) => (LLVMBuildFSub, LLVMRustSetFastMath),
473        fmul_fast(x, y) => (LLVMBuildFMul, LLVMRustSetFastMath),
474        fdiv_fast(x, y) => (LLVMBuildFDiv, LLVMRustSetFastMath),
475        frem_fast(x, y) => (LLVMBuildFRem, LLVMRustSetFastMath),
476        fadd_algebraic(x, y) => (LLVMBuildFAdd, LLVMRustSetAlgebraicMath),
477        fsub_algebraic(x, y) => (LLVMBuildFSub, LLVMRustSetAlgebraicMath),
478        fmul_algebraic(x, y) => (LLVMBuildFMul, LLVMRustSetAlgebraicMath),
479        fdiv_algebraic(x, y) => (LLVMBuildFDiv, LLVMRustSetAlgebraicMath),
480        frem_algebraic(x, y) => (LLVMBuildFRem, LLVMRustSetAlgebraicMath),
481    }
482
483    fn checked_binop(
484        &mut self,
485        oop: OverflowOp,
486        ty: Ty<'tcx>,
487        lhs: Self::Value,
488        rhs: Self::Value,
489    ) -> (Self::Value, Self::Value) {
490        let (size, signed) = ty.int_size_and_signed(self.tcx);
491        let width = size.bits();
492
493        if oop == OverflowOp::Sub && !signed {
494            // Emit sub and icmp instead of llvm.usub.with.overflow. LLVM considers these
495            // to be the canonical form. It will attempt to reform llvm.usub.with.overflow
496            // in the backend if profitable.
497            let sub = self.sub(lhs, rhs);
498            let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs);
499            return (sub, cmp);
500        }
501
502        let oop_str = match oop {
503            OverflowOp::Add => "add",
504            OverflowOp::Sub => "sub",
505            OverflowOp::Mul => "mul",
506        };
507
508        let name = format!("llvm.{}{oop_str}.with.overflow", if signed { 's' } else { 'u' });
509
510        let res = self.call_intrinsic(name, &[self.type_ix(width)], &[lhs, rhs]);
511        (self.extract_value(res, 0), self.extract_value(res, 1))
512    }
513
514    fn from_immediate(&mut self, val: Self::Value) -> Self::Value {
515        if self.cx().val_ty(val) == self.cx().type_i1() {
516            self.zext(val, self.cx().type_i8())
517        } else {
518            val
519        }
520    }
521
522    fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value {
523        if scalar.is_bool() {
524            return self.unchecked_utrunc(val, self.cx().type_i1());
525        }
526        val
527    }
528
529    fn alloca(&mut self, size: Size, align: Align) -> &'ll Value {
530        let mut bx = Builder::with_cx(self.cx);
531        bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
532        let ty = self.cx().type_array(self.cx().type_i8(), size.bytes());
533        unsafe {
534            let alloca = llvm::LLVMBuildAlloca(bx.llbuilder, ty, UNNAMED);
535            llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
536            // Cast to default addrspace if necessary
537            llvm::LLVMBuildPointerCast(bx.llbuilder, alloca, self.cx().type_ptr(), UNNAMED)
538        }
539    }
540
541    fn dynamic_alloca(&mut self, size: &'ll Value, align: Align) -> &'ll Value {
542        unsafe {
543            let alloca =
544                llvm::LLVMBuildArrayAlloca(self.llbuilder, self.cx().type_i8(), size, UNNAMED);
545            llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
546            // Cast to default addrspace if necessary
547            llvm::LLVMBuildPointerCast(self.llbuilder, alloca, self.cx().type_ptr(), UNNAMED)
548        }
549    }
550
551    fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
552        unsafe {
553            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
554            let align = align.min(self.cx().tcx.sess.target.max_reliable_alignment());
555            llvm::LLVMSetAlignment(load, align.bytes() as c_uint);
556            load
557        }
558    }
559
560    fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value {
561        unsafe {
562            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
563            llvm::LLVMSetVolatile(load, llvm::True);
564            load
565        }
566    }
567
568    fn atomic_load(
569        &mut self,
570        ty: &'ll Type,
571        ptr: &'ll Value,
572        order: rustc_middle::ty::AtomicOrdering,
573        size: Size,
574    ) -> &'ll Value {
575        unsafe {
576            let load = llvm::LLVMRustBuildAtomicLoad(
577                self.llbuilder,
578                ty,
579                ptr,
580                UNNAMED,
581                AtomicOrdering::from_generic(order),
582            );
583            // LLVM requires the alignment of atomic loads to be at least the size of the type.
584            llvm::LLVMSetAlignment(load, size.bytes() as c_uint);
585            load
586        }
587    }
588
589    #[instrument(level = "trace", skip(self))]
590    fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> {
591        if place.layout.is_unsized() {
592            let tail = self.tcx.struct_tail_for_codegen(place.layout.ty, self.typing_env());
593            if matches!(tail.kind(), ty::Foreign(..)) {
594                // Unsized locals and, at least conceptually, even unsized arguments must be copied
595                // around, which requires dynamically determining their size. Therefore, we cannot
596                // allow `extern` types here. Consult t-opsem before removing this check.
597                panic!("unsized locals must not be `extern` types");
598            }
599        }
600        assert_eq!(place.val.llextra.is_some(), place.layout.is_unsized());
601
602        if place.layout.is_zst() {
603            return OperandRef::zero_sized(place.layout);
604        }
605
606        #[instrument(level = "trace", skip(bx))]
607        fn scalar_load_metadata<'a, 'll, 'tcx>(
608            bx: &mut Builder<'a, 'll, 'tcx>,
609            load: &'ll Value,
610            scalar: abi::Scalar,
611            layout: TyAndLayout<'tcx>,
612            offset: Size,
613        ) {
614            if bx.cx.sess().opts.optimize == OptLevel::No {
615                // Don't emit metadata we're not going to use
616                return;
617            }
618
619            if !scalar.is_uninit_valid() {
620                bx.noundef_metadata(load);
621            }
622
623            match scalar.primitive() {
624                abi::Primitive::Int(..) => {
625                    if !scalar.is_always_valid(bx) {
626                        bx.range_metadata(load, scalar.valid_range(bx));
627                    }
628                }
629                abi::Primitive::Pointer(_) => {
630                    if !scalar.valid_range(bx).contains(0) {
631                        bx.nonnull_metadata(load);
632                    }
633
634                    if let Some(pointee) = layout.pointee_info_at(bx, offset) {
635                        if let Some(_) = pointee.safe {
636                            bx.align_metadata(load, pointee.align);
637                        }
638                    }
639                }
640                abi::Primitive::Float(_) => {}
641            }
642        }
643
644        let val = if let Some(_) = place.val.llextra {
645            // FIXME: Merge with the `else` below?
646            OperandValue::Ref(place.val)
647        } else if place.layout.is_llvm_immediate() {
648            let mut const_llval = None;
649            let llty = place.layout.llvm_type(self);
650            unsafe {
651                if let Some(global) = llvm::LLVMIsAGlobalVariable(place.val.llval) {
652                    if llvm::LLVMIsGlobalConstant(global) == llvm::True {
653                        if let Some(init) = llvm::LLVMGetInitializer(global) {
654                            if self.val_ty(init) == llty {
655                                const_llval = Some(init);
656                            }
657                        }
658                    }
659                }
660            }
661            let llval = const_llval.unwrap_or_else(|| {
662                let load = self.load(llty, place.val.llval, place.val.align);
663                if let abi::BackendRepr::Scalar(scalar) = place.layout.backend_repr {
664                    scalar_load_metadata(self, load, scalar, place.layout, Size::ZERO);
665                    self.to_immediate_scalar(load, scalar)
666                } else {
667                    load
668                }
669            });
670            OperandValue::Immediate(llval)
671        } else if let abi::BackendRepr::ScalarPair(a, b) = place.layout.backend_repr {
672            let b_offset = a.size(self).align_to(b.align(self).abi);
673
674            let mut load = |i, scalar: abi::Scalar, layout, align, offset| {
675                let llptr = if i == 0 {
676                    place.val.llval
677                } else {
678                    self.inbounds_ptradd(place.val.llval, self.const_usize(b_offset.bytes()))
679                };
680                let llty = place.layout.scalar_pair_element_llvm_type(self, i, false);
681                let load = self.load(llty, llptr, align);
682                scalar_load_metadata(self, load, scalar, layout, offset);
683                self.to_immediate_scalar(load, scalar)
684            };
685
686            OperandValue::Pair(
687                load(0, a, place.layout, place.val.align, Size::ZERO),
688                load(1, b, place.layout, place.val.align.restrict_for_offset(b_offset), b_offset),
689            )
690        } else {
691            OperandValue::Ref(place.val)
692        };
693
694        OperandRef { val, layout: place.layout }
695    }
696
697    fn write_operand_repeatedly(
698        &mut self,
699        cg_elem: OperandRef<'tcx, &'ll Value>,
700        count: u64,
701        dest: PlaceRef<'tcx, &'ll Value>,
702    ) {
703        let zero = self.const_usize(0);
704        let count = self.const_usize(count);
705
706        let header_bb = self.append_sibling_block("repeat_loop_header");
707        let body_bb = self.append_sibling_block("repeat_loop_body");
708        let next_bb = self.append_sibling_block("repeat_loop_next");
709
710        self.br(header_bb);
711
712        let mut header_bx = Self::build(self.cx, header_bb);
713        let i = header_bx.phi(self.val_ty(zero), &[zero], &[self.llbb()]);
714
715        let keep_going = header_bx.icmp(IntPredicate::IntULT, i, count);
716        header_bx.cond_br(keep_going, body_bb, next_bb);
717
718        let mut body_bx = Self::build(self.cx, body_bb);
719        let dest_elem = dest.project_index(&mut body_bx, i);
720        cg_elem.val.store(&mut body_bx, dest_elem);
721
722        let next = body_bx.unchecked_uadd(i, self.const_usize(1));
723        body_bx.br(header_bb);
724        header_bx.add_incoming_to_phi(i, next, body_bb);
725
726        *self = Self::build(self.cx, next_bb);
727    }
728
729    fn range_metadata(&mut self, load: &'ll Value, range: WrappingRange) {
730        if self.cx.sess().opts.optimize == OptLevel::No {
731            // Don't emit metadata we're not going to use
732            return;
733        }
734
735        unsafe {
736            let llty = self.cx.val_ty(load);
737            let md = [
738                llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.start)),
739                llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.end.wrapping_add(1))),
740            ];
741            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len());
742            self.set_metadata(load, llvm::MD_range, md);
743        }
744    }
745
746    fn nonnull_metadata(&mut self, load: &'ll Value) {
747        unsafe {
748            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
749            self.set_metadata(load, llvm::MD_nonnull, md);
750        }
751    }
752
753    fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value {
754        self.store_with_flags(val, ptr, align, MemFlags::empty())
755    }
756
757    fn store_with_flags(
758        &mut self,
759        val: &'ll Value,
760        ptr: &'ll Value,
761        align: Align,
762        flags: MemFlags,
763    ) -> &'ll Value {
764        debug!("Store {:?} -> {:?} ({:?})", val, ptr, flags);
765        assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
766        unsafe {
767            let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
768            let align = align.min(self.cx().tcx.sess.target.max_reliable_alignment());
769            let align =
770                if flags.contains(MemFlags::UNALIGNED) { 1 } else { align.bytes() as c_uint };
771            llvm::LLVMSetAlignment(store, align);
772            if flags.contains(MemFlags::VOLATILE) {
773                llvm::LLVMSetVolatile(store, llvm::True);
774            }
775            if flags.contains(MemFlags::NONTEMPORAL) {
776                // Make sure that the current target architectures supports "sane" non-temporal
777                // stores, i.e., non-temporal stores that are equivalent to regular stores except
778                // for performance. LLVM doesn't seem to care about this, and will happily treat
779                // `!nontemporal` stores as-if they were normal stores (for reordering optimizations
780                // etc) even on x86, despite later lowering them to MOVNT which do *not* behave like
781                // regular stores but require special fences. So we keep a list of architectures
782                // where `!nontemporal` is known to be truly just a hint, and use regular stores
783                // everywhere else. (In the future, we could alternatively ensure that an sfence
784                // gets emitted after a sequence of movnt before any kind of synchronizing
785                // operation. But it's not clear how to do that with LLVM.)
786                // For more context, see <https://github.com/rust-lang/rust/issues/114582> and
787                // <https://github.com/llvm/llvm-project/issues/64521>.
788                const WELL_BEHAVED_NONTEMPORAL_ARCHS: &[&str] =
789                    &["aarch64", "arm", "riscv32", "riscv64"];
790
791                let use_nontemporal =
792                    WELL_BEHAVED_NONTEMPORAL_ARCHS.contains(&&*self.cx.tcx.sess.target.arch);
793                if use_nontemporal {
794                    // According to LLVM [1] building a nontemporal store must
795                    // *always* point to a metadata value of the integer 1.
796                    //
797                    // [1]: https://llvm.org/docs/LangRef.html#store-instruction
798                    let one = llvm::LLVMValueAsMetadata(self.cx.const_i32(1));
799                    let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, &one, 1);
800                    self.set_metadata(store, llvm::MD_nontemporal, md);
801                }
802            }
803            store
804        }
805    }
806
807    fn atomic_store(
808        &mut self,
809        val: &'ll Value,
810        ptr: &'ll Value,
811        order: rustc_middle::ty::AtomicOrdering,
812        size: Size,
813    ) {
814        debug!("Store {:?} -> {:?}", val, ptr);
815        assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
816        unsafe {
817            let store = llvm::LLVMRustBuildAtomicStore(
818                self.llbuilder,
819                val,
820                ptr,
821                AtomicOrdering::from_generic(order),
822            );
823            // LLVM requires the alignment of atomic stores to be at least the size of the type.
824            llvm::LLVMSetAlignment(store, size.bytes() as c_uint);
825        }
826    }
827
828    fn gep(&mut self, ty: &'ll Type, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value {
829        unsafe {
830            llvm::LLVMBuildGEPWithNoWrapFlags(
831                self.llbuilder,
832                ty,
833                ptr,
834                indices.as_ptr(),
835                indices.len() as c_uint,
836                UNNAMED,
837                GEPNoWrapFlags::default(),
838            )
839        }
840    }
841
842    fn inbounds_gep(
843        &mut self,
844        ty: &'ll Type,
845        ptr: &'ll Value,
846        indices: &[&'ll Value],
847    ) -> &'ll Value {
848        unsafe {
849            llvm::LLVMBuildGEPWithNoWrapFlags(
850                self.llbuilder,
851                ty,
852                ptr,
853                indices.as_ptr(),
854                indices.len() as c_uint,
855                UNNAMED,
856                GEPNoWrapFlags::InBounds,
857            )
858        }
859    }
860
861    fn inbounds_nuw_gep(
862        &mut self,
863        ty: &'ll Type,
864        ptr: &'ll Value,
865        indices: &[&'ll Value],
866    ) -> &'ll Value {
867        unsafe {
868            llvm::LLVMBuildGEPWithNoWrapFlags(
869                self.llbuilder,
870                ty,
871                ptr,
872                indices.as_ptr(),
873                indices.len() as c_uint,
874                UNNAMED,
875                GEPNoWrapFlags::InBounds | GEPNoWrapFlags::NUW,
876            )
877        }
878    }
879
880    /* Casts */
881    fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
882        unsafe { llvm::LLVMBuildTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
883    }
884
885    fn unchecked_utrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
886        debug_assert_ne!(self.val_ty(val), dest_ty);
887
888        let trunc = self.trunc(val, dest_ty);
889        unsafe {
890            if llvm::LLVMIsAInstruction(trunc).is_some() {
891                llvm::LLVMSetNUW(trunc, True);
892            }
893        }
894        trunc
895    }
896
897    fn unchecked_strunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
898        debug_assert_ne!(self.val_ty(val), dest_ty);
899
900        let trunc = self.trunc(val, dest_ty);
901        unsafe {
902            if llvm::LLVMIsAInstruction(trunc).is_some() {
903                llvm::LLVMSetNSW(trunc, True);
904            }
905        }
906        trunc
907    }
908
909    fn sext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
910        unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, UNNAMED) }
911    }
912
913    fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
914        self.call_intrinsic("llvm.fptoui.sat", &[dest_ty, self.val_ty(val)], &[val])
915    }
916
917    fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
918        self.call_intrinsic("llvm.fptosi.sat", &[dest_ty, self.val_ty(val)], &[val])
919    }
920
921    fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
922        // On WebAssembly the `fptoui` and `fptosi` instructions currently have
923        // poor codegen. The reason for this is that the corresponding wasm
924        // instructions, `i32.trunc_f32_s` for example, will trap when the float
925        // is out-of-bounds, infinity, or nan. This means that LLVM
926        // automatically inserts control flow around `fptoui` and `fptosi`
927        // because the LLVM instruction `fptoui` is defined as producing a
928        // poison value, not having UB on out-of-bounds values.
929        //
930        // This method, however, is only used with non-saturating casts that
931        // have UB on out-of-bounds values. This means that it's ok if we use
932        // the raw wasm instruction since out-of-bounds values can do whatever
933        // we like. To ensure that LLVM picks the right instruction we choose
934        // the raw wasm intrinsic functions which avoid LLVM inserting all the
935        // other control flow automatically.
936        if self.sess().target.is_like_wasm {
937            let src_ty = self.cx.val_ty(val);
938            if self.cx.type_kind(src_ty) != TypeKind::Vector {
939                let float_width = self.cx.float_width(src_ty);
940                let int_width = self.cx.int_width(dest_ty);
941                if matches!((int_width, float_width), (32 | 64, 32 | 64)) {
942                    return self.call_intrinsic(
943                        "llvm.wasm.trunc.unsigned",
944                        &[dest_ty, src_ty],
945                        &[val],
946                    );
947                }
948            }
949        }
950        unsafe { llvm::LLVMBuildFPToUI(self.llbuilder, val, dest_ty, UNNAMED) }
951    }
952
953    fn fptosi(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
954        // see `fptoui` above for why wasm is different here
955        if self.sess().target.is_like_wasm {
956            let src_ty = self.cx.val_ty(val);
957            if self.cx.type_kind(src_ty) != TypeKind::Vector {
958                let float_width = self.cx.float_width(src_ty);
959                let int_width = self.cx.int_width(dest_ty);
960                if matches!((int_width, float_width), (32 | 64, 32 | 64)) {
961                    return self.call_intrinsic(
962                        "llvm.wasm.trunc.signed",
963                        &[dest_ty, src_ty],
964                        &[val],
965                    );
966                }
967            }
968        }
969        unsafe { llvm::LLVMBuildFPToSI(self.llbuilder, val, dest_ty, UNNAMED) }
970    }
971
972    fn uitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
973        unsafe { llvm::LLVMBuildUIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
974    }
975
976    fn sitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
977        unsafe { llvm::LLVMBuildSIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
978    }
979
980    fn fptrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
981        unsafe { llvm::LLVMBuildFPTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
982    }
983
984    fn fpext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
985        unsafe { llvm::LLVMBuildFPExt(self.llbuilder, val, dest_ty, UNNAMED) }
986    }
987
988    fn ptrtoint(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
989        unsafe { llvm::LLVMBuildPtrToInt(self.llbuilder, val, dest_ty, UNNAMED) }
990    }
991
992    fn inttoptr(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
993        unsafe { llvm::LLVMBuildIntToPtr(self.llbuilder, val, dest_ty, UNNAMED) }
994    }
995
996    fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
997        unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) }
998    }
999
1000    fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value {
1001        unsafe {
1002            llvm::LLVMBuildIntCast2(
1003                self.llbuilder,
1004                val,
1005                dest_ty,
1006                if is_signed { True } else { False },
1007                UNNAMED,
1008            )
1009        }
1010    }
1011
1012    fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1013        unsafe { llvm::LLVMBuildPointerCast(self.llbuilder, val, dest_ty, UNNAMED) }
1014    }
1015
1016    /* Comparisons */
1017    fn icmp(&mut self, op: IntPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1018        let op = llvm::IntPredicate::from_generic(op);
1019        unsafe { llvm::LLVMBuildICmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
1020    }
1021
1022    fn fcmp(&mut self, op: RealPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1023        let op = llvm::RealPredicate::from_generic(op);
1024        unsafe { llvm::LLVMBuildFCmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
1025    }
1026
1027    fn three_way_compare(
1028        &mut self,
1029        ty: Ty<'tcx>,
1030        lhs: Self::Value,
1031        rhs: Self::Value,
1032    ) -> Option<Self::Value> {
1033        // FIXME: See comment on the definition of `three_way_compare`.
1034        if crate::llvm_util::get_version() < (20, 0, 0) {
1035            return None;
1036        }
1037
1038        let size = ty.primitive_size(self.tcx);
1039        let name = if ty.is_signed() { "llvm.scmp" } else { "llvm.ucmp" };
1040
1041        Some(self.call_intrinsic(name, &[self.type_i8(), self.type_ix(size.bits())], &[lhs, rhs]))
1042    }
1043
1044    /* Miscellaneous instructions */
1045    fn memcpy(
1046        &mut self,
1047        dst: &'ll Value,
1048        dst_align: Align,
1049        src: &'ll Value,
1050        src_align: Align,
1051        size: &'ll Value,
1052        flags: MemFlags,
1053    ) {
1054        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported");
1055        let size = self.intcast(size, self.type_isize(), false);
1056        let is_volatile = flags.contains(MemFlags::VOLATILE);
1057        unsafe {
1058            llvm::LLVMRustBuildMemCpy(
1059                self.llbuilder,
1060                dst,
1061                dst_align.bytes() as c_uint,
1062                src,
1063                src_align.bytes() as c_uint,
1064                size,
1065                is_volatile,
1066            );
1067        }
1068    }
1069
1070    fn memmove(
1071        &mut self,
1072        dst: &'ll Value,
1073        dst_align: Align,
1074        src: &'ll Value,
1075        src_align: Align,
1076        size: &'ll Value,
1077        flags: MemFlags,
1078    ) {
1079        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memmove not supported");
1080        let size = self.intcast(size, self.type_isize(), false);
1081        let is_volatile = flags.contains(MemFlags::VOLATILE);
1082        unsafe {
1083            llvm::LLVMRustBuildMemMove(
1084                self.llbuilder,
1085                dst,
1086                dst_align.bytes() as c_uint,
1087                src,
1088                src_align.bytes() as c_uint,
1089                size,
1090                is_volatile,
1091            );
1092        }
1093    }
1094
1095    fn memset(
1096        &mut self,
1097        ptr: &'ll Value,
1098        fill_byte: &'ll Value,
1099        size: &'ll Value,
1100        align: Align,
1101        flags: MemFlags,
1102    ) {
1103        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memset not supported");
1104        let is_volatile = flags.contains(MemFlags::VOLATILE);
1105        unsafe {
1106            llvm::LLVMRustBuildMemSet(
1107                self.llbuilder,
1108                ptr,
1109                align.bytes() as c_uint,
1110                fill_byte,
1111                size,
1112                is_volatile,
1113            );
1114        }
1115    }
1116
1117    fn select(
1118        &mut self,
1119        cond: &'ll Value,
1120        then_val: &'ll Value,
1121        else_val: &'ll Value,
1122    ) -> &'ll Value {
1123        unsafe { llvm::LLVMBuildSelect(self.llbuilder, cond, then_val, else_val, UNNAMED) }
1124    }
1125
1126    fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1127        unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
1128    }
1129
1130    fn extract_element(&mut self, vec: &'ll Value, idx: &'ll Value) -> &'ll Value {
1131        unsafe { llvm::LLVMBuildExtractElement(self.llbuilder, vec, idx, UNNAMED) }
1132    }
1133
1134    fn vector_splat(&mut self, num_elts: usize, elt: &'ll Value) -> &'ll Value {
1135        unsafe {
1136            let elt_ty = self.cx.val_ty(elt);
1137            let undef = llvm::LLVMGetUndef(self.type_vector(elt_ty, num_elts as u64));
1138            let vec = self.insert_element(undef, elt, self.cx.const_i32(0));
1139            let vec_i32_ty = self.type_vector(self.type_i32(), num_elts as u64);
1140            self.shuffle_vector(vec, undef, self.const_null(vec_i32_ty))
1141        }
1142    }
1143
1144    fn extract_value(&mut self, agg_val: &'ll Value, idx: u64) -> &'ll Value {
1145        assert_eq!(idx as c_uint as u64, idx);
1146        unsafe { llvm::LLVMBuildExtractValue(self.llbuilder, agg_val, idx as c_uint, UNNAMED) }
1147    }
1148
1149    fn insert_value(&mut self, agg_val: &'ll Value, elt: &'ll Value, idx: u64) -> &'ll Value {
1150        assert_eq!(idx as c_uint as u64, idx);
1151        unsafe { llvm::LLVMBuildInsertValue(self.llbuilder, agg_val, elt, idx as c_uint, UNNAMED) }
1152    }
1153
1154    fn set_personality_fn(&mut self, personality: &'ll Value) {
1155        unsafe {
1156            llvm::LLVMSetPersonalityFn(self.llfn(), personality);
1157        }
1158    }
1159
1160    fn cleanup_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) {
1161        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1162        let landing_pad = self.landing_pad(ty, pers_fn, 0);
1163        unsafe {
1164            llvm::LLVMSetCleanup(landing_pad, llvm::True);
1165        }
1166        (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1))
1167    }
1168
1169    fn filter_landing_pad(&mut self, pers_fn: &'ll Value) {
1170        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1171        let landing_pad = self.landing_pad(ty, pers_fn, 1);
1172        self.add_clause(landing_pad, self.const_array(self.type_ptr(), &[]));
1173    }
1174
1175    fn resume(&mut self, exn0: &'ll Value, exn1: &'ll Value) {
1176        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1177        let mut exn = self.const_poison(ty);
1178        exn = self.insert_value(exn, exn0, 0);
1179        exn = self.insert_value(exn, exn1, 1);
1180        unsafe {
1181            llvm::LLVMBuildResume(self.llbuilder, exn);
1182        }
1183    }
1184
1185    fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> {
1186        let ret = unsafe {
1187            llvm::LLVMBuildCleanupPad(
1188                self.llbuilder,
1189                parent,
1190                args.as_ptr(),
1191                args.len() as c_uint,
1192                c"cleanuppad".as_ptr(),
1193            )
1194        };
1195        Funclet::new(ret.expect("LLVM does not have support for cleanuppad"))
1196    }
1197
1198    fn cleanup_ret(&mut self, funclet: &Funclet<'ll>, unwind: Option<&'ll BasicBlock>) {
1199        unsafe {
1200            llvm::LLVMBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind)
1201                .expect("LLVM does not have support for cleanupret");
1202        }
1203    }
1204
1205    fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> {
1206        let ret = unsafe {
1207            llvm::LLVMBuildCatchPad(
1208                self.llbuilder,
1209                parent,
1210                args.as_ptr(),
1211                args.len() as c_uint,
1212                c"catchpad".as_ptr(),
1213            )
1214        };
1215        Funclet::new(ret.expect("LLVM does not have support for catchpad"))
1216    }
1217
1218    fn catch_switch(
1219        &mut self,
1220        parent: Option<&'ll Value>,
1221        unwind: Option<&'ll BasicBlock>,
1222        handlers: &[&'ll BasicBlock],
1223    ) -> &'ll Value {
1224        let ret = unsafe {
1225            llvm::LLVMBuildCatchSwitch(
1226                self.llbuilder,
1227                parent,
1228                unwind,
1229                handlers.len() as c_uint,
1230                c"catchswitch".as_ptr(),
1231            )
1232        };
1233        let ret = ret.expect("LLVM does not have support for catchswitch");
1234        for handler in handlers {
1235            unsafe {
1236                llvm::LLVMAddHandler(ret, handler);
1237            }
1238        }
1239        ret
1240    }
1241
1242    // Atomic Operations
1243    fn atomic_cmpxchg(
1244        &mut self,
1245        dst: &'ll Value,
1246        cmp: &'ll Value,
1247        src: &'ll Value,
1248        order: rustc_middle::ty::AtomicOrdering,
1249        failure_order: rustc_middle::ty::AtomicOrdering,
1250        weak: bool,
1251    ) -> (&'ll Value, &'ll Value) {
1252        let weak = if weak { llvm::True } else { llvm::False };
1253        unsafe {
1254            let value = llvm::LLVMBuildAtomicCmpXchg(
1255                self.llbuilder,
1256                dst,
1257                cmp,
1258                src,
1259                AtomicOrdering::from_generic(order),
1260                AtomicOrdering::from_generic(failure_order),
1261                llvm::False, // SingleThreaded
1262            );
1263            llvm::LLVMSetWeak(value, weak);
1264            let val = self.extract_value(value, 0);
1265            let success = self.extract_value(value, 1);
1266            (val, success)
1267        }
1268    }
1269
1270    fn atomic_rmw(
1271        &mut self,
1272        op: rustc_codegen_ssa::common::AtomicRmwBinOp,
1273        dst: &'ll Value,
1274        mut src: &'ll Value,
1275        order: rustc_middle::ty::AtomicOrdering,
1276    ) -> &'ll Value {
1277        // The only RMW operation that LLVM supports on pointers is compare-exchange.
1278        let requires_cast_to_int = self.val_ty(src) == self.type_ptr()
1279            && op != rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicXchg;
1280        if requires_cast_to_int {
1281            src = self.ptrtoint(src, self.type_isize());
1282        }
1283        let mut res = unsafe {
1284            llvm::LLVMBuildAtomicRMW(
1285                self.llbuilder,
1286                AtomicRmwBinOp::from_generic(op),
1287                dst,
1288                src,
1289                AtomicOrdering::from_generic(order),
1290                llvm::False, // SingleThreaded
1291            )
1292        };
1293        if requires_cast_to_int {
1294            res = self.inttoptr(res, self.type_ptr());
1295        }
1296        res
1297    }
1298
1299    fn atomic_fence(
1300        &mut self,
1301        order: rustc_middle::ty::AtomicOrdering,
1302        scope: SynchronizationScope,
1303    ) {
1304        let single_threaded = match scope {
1305            SynchronizationScope::SingleThread => llvm::True,
1306            SynchronizationScope::CrossThread => llvm::False,
1307        };
1308        unsafe {
1309            llvm::LLVMBuildFence(
1310                self.llbuilder,
1311                AtomicOrdering::from_generic(order),
1312                single_threaded,
1313                UNNAMED,
1314            );
1315        }
1316    }
1317
1318    fn set_invariant_load(&mut self, load: &'ll Value) {
1319        unsafe {
1320            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
1321            self.set_metadata(load, llvm::MD_invariant_load, md);
1322        }
1323    }
1324
1325    fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) {
1326        self.call_lifetime_intrinsic("llvm.lifetime.start", ptr, size);
1327    }
1328
1329    fn lifetime_end(&mut self, ptr: &'ll Value, size: Size) {
1330        self.call_lifetime_intrinsic("llvm.lifetime.end", ptr, size);
1331    }
1332
1333    fn call(
1334        &mut self,
1335        llty: &'ll Type,
1336        fn_attrs: Option<&CodegenFnAttrs>,
1337        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1338        llfn: &'ll Value,
1339        args: &[&'ll Value],
1340        funclet: Option<&Funclet<'ll>>,
1341        instance: Option<Instance<'tcx>>,
1342    ) -> &'ll Value {
1343        debug!("call {:?} with args ({:?})", llfn, args);
1344
1345        let args = self.check_call("call", llty, llfn, args);
1346        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1347        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1348        if let Some(funclet_bundle) = funclet_bundle {
1349            bundles.push(funclet_bundle);
1350        }
1351
1352        // Emit CFI pointer type membership test
1353        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
1354
1355        // Emit KCFI operand bundle
1356        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
1357        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
1358            bundles.push(kcfi_bundle);
1359        }
1360
1361        let call = unsafe {
1362            llvm::LLVMBuildCallWithOperandBundles(
1363                self.llbuilder,
1364                llty,
1365                llfn,
1366                args.as_ptr() as *const &llvm::Value,
1367                args.len() as c_uint,
1368                bundles.as_ptr(),
1369                bundles.len() as c_uint,
1370                c"".as_ptr(),
1371            )
1372        };
1373        if let Some(fn_abi) = fn_abi {
1374            fn_abi.apply_attrs_callsite(self, call);
1375        }
1376        call
1377    }
1378
1379    fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1380        unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) }
1381    }
1382
1383    fn apply_attrs_to_cleanup_callsite(&mut self, llret: &'ll Value) {
1384        // Cleanup is always the cold path.
1385        let cold_inline = llvm::AttributeKind::Cold.create_attr(self.llcx);
1386        attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_inline]);
1387    }
1388}
1389
1390impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> {
1391    fn get_static(&mut self, def_id: DefId) -> &'ll Value {
1392        // Forward to the `get_static` method of `CodegenCx`
1393        let global = self.cx().get_static(def_id);
1394        if self.cx().tcx.is_thread_local_static(def_id) {
1395            let pointer =
1396                self.call_intrinsic("llvm.threadlocal.address", &[self.val_ty(global)], &[global]);
1397            // Cast to default address space if globals are in a different addrspace
1398            self.pointercast(pointer, self.type_ptr())
1399        } else {
1400            // Cast to default address space if globals are in a different addrspace
1401            self.cx().const_pointercast(global, self.type_ptr())
1402        }
1403    }
1404}
1405
1406impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1407    pub(crate) fn llfn(&self) -> &'ll Value {
1408        unsafe { llvm::LLVMGetBasicBlockParent(self.llbb()) }
1409    }
1410}
1411
1412impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1413    fn position_at_start(&mut self, llbb: &'ll BasicBlock) {
1414        unsafe {
1415            llvm::LLVMRustPositionBuilderAtStart(self.llbuilder, llbb);
1416        }
1417    }
1418}
1419impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1420    fn align_metadata(&mut self, load: &'ll Value, align: Align) {
1421        unsafe {
1422            let md = [llvm::LLVMValueAsMetadata(self.cx.const_u64(align.bytes()))];
1423            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len());
1424            self.set_metadata(load, llvm::MD_align, md);
1425        }
1426    }
1427
1428    fn noundef_metadata(&mut self, load: &'ll Value) {
1429        unsafe {
1430            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
1431            self.set_metadata(load, llvm::MD_noundef, md);
1432        }
1433    }
1434
1435    pub(crate) fn set_unpredictable(&mut self, inst: &'ll Value) {
1436        unsafe {
1437            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
1438            self.set_metadata(inst, llvm::MD_unpredictable, md);
1439        }
1440    }
1441}
1442impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1443    pub(crate) fn minnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1444        unsafe { llvm::LLVMRustBuildMinNum(self.llbuilder, lhs, rhs) }
1445    }
1446
1447    pub(crate) fn maxnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1448        unsafe { llvm::LLVMRustBuildMaxNum(self.llbuilder, lhs, rhs) }
1449    }
1450
1451    pub(crate) fn insert_element(
1452        &mut self,
1453        vec: &'ll Value,
1454        elt: &'ll Value,
1455        idx: &'ll Value,
1456    ) -> &'ll Value {
1457        unsafe { llvm::LLVMBuildInsertElement(self.llbuilder, vec, elt, idx, UNNAMED) }
1458    }
1459
1460    pub(crate) fn shuffle_vector(
1461        &mut self,
1462        v1: &'ll Value,
1463        v2: &'ll Value,
1464        mask: &'ll Value,
1465    ) -> &'ll Value {
1466        unsafe { llvm::LLVMBuildShuffleVector(self.llbuilder, v1, v2, mask, UNNAMED) }
1467    }
1468
1469    pub(crate) fn vector_reduce_fadd(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1470        unsafe { llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src) }
1471    }
1472    pub(crate) fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1473        unsafe { llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src) }
1474    }
1475    pub(crate) fn vector_reduce_fadd_reassoc(
1476        &mut self,
1477        acc: &'ll Value,
1478        src: &'ll Value,
1479    ) -> &'ll Value {
1480        unsafe {
1481            let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src);
1482            llvm::LLVMRustSetAllowReassoc(instr);
1483            instr
1484        }
1485    }
1486    pub(crate) fn vector_reduce_fmul_reassoc(
1487        &mut self,
1488        acc: &'ll Value,
1489        src: &'ll Value,
1490    ) -> &'ll Value {
1491        unsafe {
1492            let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src);
1493            llvm::LLVMRustSetAllowReassoc(instr);
1494            instr
1495        }
1496    }
1497    pub(crate) fn vector_reduce_add(&mut self, src: &'ll Value) -> &'ll Value {
1498        unsafe { llvm::LLVMRustBuildVectorReduceAdd(self.llbuilder, src) }
1499    }
1500    pub(crate) fn vector_reduce_mul(&mut self, src: &'ll Value) -> &'ll Value {
1501        unsafe { llvm::LLVMRustBuildVectorReduceMul(self.llbuilder, src) }
1502    }
1503    pub(crate) fn vector_reduce_and(&mut self, src: &'ll Value) -> &'ll Value {
1504        unsafe { llvm::LLVMRustBuildVectorReduceAnd(self.llbuilder, src) }
1505    }
1506    pub(crate) fn vector_reduce_or(&mut self, src: &'ll Value) -> &'ll Value {
1507        unsafe { llvm::LLVMRustBuildVectorReduceOr(self.llbuilder, src) }
1508    }
1509    pub(crate) fn vector_reduce_xor(&mut self, src: &'ll Value) -> &'ll Value {
1510        unsafe { llvm::LLVMRustBuildVectorReduceXor(self.llbuilder, src) }
1511    }
1512    pub(crate) fn vector_reduce_fmin(&mut self, src: &'ll Value) -> &'ll Value {
1513        unsafe {
1514            llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ false)
1515        }
1516    }
1517    pub(crate) fn vector_reduce_fmax(&mut self, src: &'ll Value) -> &'ll Value {
1518        unsafe {
1519            llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ false)
1520        }
1521    }
1522    pub(crate) fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1523        unsafe { llvm::LLVMRustBuildVectorReduceMin(self.llbuilder, src, is_signed) }
1524    }
1525    pub(crate) fn vector_reduce_max(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1526        unsafe { llvm::LLVMRustBuildVectorReduceMax(self.llbuilder, src, is_signed) }
1527    }
1528
1529    pub(crate) fn add_clause(&mut self, landing_pad: &'ll Value, clause: &'ll Value) {
1530        unsafe {
1531            llvm::LLVMAddClause(landing_pad, clause);
1532        }
1533    }
1534
1535    pub(crate) fn catch_ret(
1536        &mut self,
1537        funclet: &Funclet<'ll>,
1538        unwind: &'ll BasicBlock,
1539    ) -> &'ll Value {
1540        let ret = unsafe { llvm::LLVMBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) };
1541        ret.expect("LLVM does not have support for catchret")
1542    }
1543
1544    fn check_call<'b>(
1545        &mut self,
1546        typ: &str,
1547        fn_ty: &'ll Type,
1548        llfn: &'ll Value,
1549        args: &'b [&'ll Value],
1550    ) -> Cow<'b, [&'ll Value]> {
1551        assert!(
1552            self.cx.type_kind(fn_ty) == TypeKind::Function,
1553            "builder::{typ} not passed a function, but {fn_ty:?}"
1554        );
1555
1556        let param_tys = self.cx.func_params_types(fn_ty);
1557
1558        let all_args_match = iter::zip(&param_tys, args.iter().map(|&v| self.cx.val_ty(v)))
1559            .all(|(expected_ty, actual_ty)| *expected_ty == actual_ty);
1560
1561        if all_args_match {
1562            return Cow::Borrowed(args);
1563        }
1564
1565        let casted_args: Vec<_> = iter::zip(param_tys, args)
1566            .enumerate()
1567            .map(|(i, (expected_ty, &actual_val))| {
1568                let actual_ty = self.cx.val_ty(actual_val);
1569                if expected_ty != actual_ty {
1570                    debug!(
1571                        "type mismatch in function call of {:?}. \
1572                            Expected {:?} for param {}, got {:?}; injecting bitcast",
1573                        llfn, expected_ty, i, actual_ty
1574                    );
1575                    self.bitcast(actual_val, expected_ty)
1576                } else {
1577                    actual_val
1578                }
1579            })
1580            .collect();
1581
1582        Cow::Owned(casted_args)
1583    }
1584
1585    pub(crate) fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1586        unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
1587    }
1588}
1589
1590impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1591    pub(crate) fn call_intrinsic(
1592        &mut self,
1593        base_name: impl Into<Cow<'static, str>>,
1594        type_params: &[&'ll Type],
1595        args: &[&'ll Value],
1596    ) -> &'ll Value {
1597        let (ty, f) = self.cx.get_intrinsic(base_name.into(), type_params);
1598        self.call(ty, None, None, f, args, None, None)
1599    }
1600
1601    fn call_lifetime_intrinsic(&mut self, intrinsic: &'static str, ptr: &'ll Value, size: Size) {
1602        let size = size.bytes();
1603        if size == 0 {
1604            return;
1605        }
1606
1607        if !self.cx().sess().emit_lifetime_markers() {
1608            return;
1609        }
1610
1611        self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[self.cx.const_u64(size), ptr]);
1612    }
1613}
1614impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1615    pub(crate) fn phi(
1616        &mut self,
1617        ty: &'ll Type,
1618        vals: &[&'ll Value],
1619        bbs: &[&'ll BasicBlock],
1620    ) -> &'ll Value {
1621        assert_eq!(vals.len(), bbs.len());
1622        let phi = unsafe { llvm::LLVMBuildPhi(self.llbuilder, ty, UNNAMED) };
1623        unsafe {
1624            llvm::LLVMAddIncoming(phi, vals.as_ptr(), bbs.as_ptr(), vals.len() as c_uint);
1625            phi
1626        }
1627    }
1628
1629    fn add_incoming_to_phi(&mut self, phi: &'ll Value, val: &'ll Value, bb: &'ll BasicBlock) {
1630        unsafe {
1631            llvm::LLVMAddIncoming(phi, &val, &bb, 1 as c_uint);
1632        }
1633    }
1634}
1635impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1636    pub(crate) fn landing_pad(
1637        &mut self,
1638        ty: &'ll Type,
1639        pers_fn: &'ll Value,
1640        num_clauses: usize,
1641    ) -> &'ll Value {
1642        // Use LLVMSetPersonalityFn to set the personality. It supports arbitrary Consts while,
1643        // LLVMBuildLandingPad requires the argument to be a Function (as of LLVM 12). The
1644        // personality lives on the parent function anyway.
1645        self.set_personality_fn(pers_fn);
1646        unsafe {
1647            llvm::LLVMBuildLandingPad(self.llbuilder, ty, None, num_clauses as c_uint, UNNAMED)
1648        }
1649    }
1650
1651    pub(crate) fn callbr(
1652        &mut self,
1653        llty: &'ll Type,
1654        fn_attrs: Option<&CodegenFnAttrs>,
1655        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1656        llfn: &'ll Value,
1657        args: &[&'ll Value],
1658        default_dest: &'ll BasicBlock,
1659        indirect_dest: &[&'ll BasicBlock],
1660        funclet: Option<&Funclet<'ll>>,
1661        instance: Option<Instance<'tcx>>,
1662    ) -> &'ll Value {
1663        debug!("invoke {:?} with args ({:?})", llfn, args);
1664
1665        let args = self.check_call("callbr", llty, llfn, args);
1666        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1667        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1668        if let Some(funclet_bundle) = funclet_bundle {
1669            bundles.push(funclet_bundle);
1670        }
1671
1672        // Emit CFI pointer type membership test
1673        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
1674
1675        // Emit KCFI operand bundle
1676        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
1677        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
1678            bundles.push(kcfi_bundle);
1679        }
1680
1681        let callbr = unsafe {
1682            llvm::LLVMBuildCallBr(
1683                self.llbuilder,
1684                llty,
1685                llfn,
1686                default_dest,
1687                indirect_dest.as_ptr(),
1688                indirect_dest.len() as c_uint,
1689                args.as_ptr(),
1690                args.len() as c_uint,
1691                bundles.as_ptr(),
1692                bundles.len() as c_uint,
1693                UNNAMED,
1694            )
1695        };
1696        if let Some(fn_abi) = fn_abi {
1697            fn_abi.apply_attrs_callsite(self, callbr);
1698        }
1699        callbr
1700    }
1701
1702    // Emits CFI pointer type membership tests.
1703    fn cfi_type_test(
1704        &mut self,
1705        fn_attrs: Option<&CodegenFnAttrs>,
1706        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1707        instance: Option<Instance<'tcx>>,
1708        llfn: &'ll Value,
1709    ) {
1710        let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
1711        if self.tcx.sess.is_sanitizer_cfi_enabled()
1712            && let Some(fn_abi) = fn_abi
1713            && is_indirect_call
1714        {
1715            if let Some(fn_attrs) = fn_attrs
1716                && fn_attrs.no_sanitize.contains(SanitizerSet::CFI)
1717            {
1718                return;
1719            }
1720
1721            let mut options = cfi::TypeIdOptions::empty();
1722            if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
1723                options.insert(cfi::TypeIdOptions::GENERALIZE_POINTERS);
1724            }
1725            if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
1726                options.insert(cfi::TypeIdOptions::NORMALIZE_INTEGERS);
1727            }
1728
1729            let typeid = if let Some(instance) = instance {
1730                cfi::typeid_for_instance(self.tcx, instance, options)
1731            } else {
1732                cfi::typeid_for_fnabi(self.tcx, fn_abi, options)
1733            };
1734            let typeid_metadata = self.cx.typeid_metadata(typeid).unwrap();
1735            let dbg_loc = self.get_dbg_loc();
1736
1737            // Test whether the function pointer is associated with the type identifier using the
1738            // llvm.type.test intrinsic. The LowerTypeTests link-time optimization pass replaces
1739            // calls to this intrinsic with code to test type membership.
1740            let typeid = self.get_metadata_value(typeid_metadata);
1741            let cond = self.call_intrinsic("llvm.type.test", &[], &[llfn, typeid]);
1742            let bb_pass = self.append_sibling_block("type_test.pass");
1743            let bb_fail = self.append_sibling_block("type_test.fail");
1744            self.cond_br(cond, bb_pass, bb_fail);
1745
1746            self.switch_to_block(bb_fail);
1747            if let Some(dbg_loc) = dbg_loc {
1748                self.set_dbg_loc(dbg_loc);
1749            }
1750            self.abort();
1751            self.unreachable();
1752
1753            self.switch_to_block(bb_pass);
1754            if let Some(dbg_loc) = dbg_loc {
1755                self.set_dbg_loc(dbg_loc);
1756            }
1757        }
1758    }
1759
1760    // Emits KCFI operand bundles.
1761    fn kcfi_operand_bundle(
1762        &mut self,
1763        fn_attrs: Option<&CodegenFnAttrs>,
1764        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1765        instance: Option<Instance<'tcx>>,
1766        llfn: &'ll Value,
1767    ) -> Option<llvm::OperandBundleBox<'ll>> {
1768        let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
1769        let kcfi_bundle = if self.tcx.sess.is_sanitizer_kcfi_enabled()
1770            && let Some(fn_abi) = fn_abi
1771            && is_indirect_call
1772        {
1773            if let Some(fn_attrs) = fn_attrs
1774                && fn_attrs.no_sanitize.contains(SanitizerSet::KCFI)
1775            {
1776                return None;
1777            }
1778
1779            let mut options = kcfi::TypeIdOptions::empty();
1780            if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
1781                options.insert(kcfi::TypeIdOptions::GENERALIZE_POINTERS);
1782            }
1783            if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
1784                options.insert(kcfi::TypeIdOptions::NORMALIZE_INTEGERS);
1785            }
1786
1787            let kcfi_typeid = if let Some(instance) = instance {
1788                kcfi::typeid_for_instance(self.tcx, instance, options)
1789            } else {
1790                kcfi::typeid_for_fnabi(self.tcx, fn_abi, options)
1791            };
1792
1793            Some(llvm::OperandBundleBox::new("kcfi", &[self.const_u32(kcfi_typeid)]))
1794        } else {
1795            None
1796        };
1797        kcfi_bundle
1798    }
1799
1800    /// Emits a call to `llvm.instrprof.increment`. Used by coverage instrumentation.
1801    #[instrument(level = "debug", skip(self))]
1802    pub(crate) fn instrprof_increment(
1803        &mut self,
1804        fn_name: &'ll Value,
1805        hash: &'ll Value,
1806        num_counters: &'ll Value,
1807        index: &'ll Value,
1808    ) {
1809        self.call_intrinsic("llvm.instrprof.increment", &[], &[fn_name, hash, num_counters, index]);
1810    }
1811
1812    /// Emits a call to `llvm.instrprof.mcdc.parameters`.
1813    ///
1814    /// This doesn't produce any code directly, but is used as input by
1815    /// the LLVM pass that handles coverage instrumentation.
1816    ///
1817    /// (See clang's [`CodeGenPGO::emitMCDCParameters`] for comparison.)
1818    ///
1819    /// [`CodeGenPGO::emitMCDCParameters`]:
1820    ///     https://github.com/rust-lang/llvm-project/blob/5399a24/clang/lib/CodeGen/CodeGenPGO.cpp#L1124
1821    #[instrument(level = "debug", skip(self))]
1822    pub(crate) fn mcdc_parameters(
1823        &mut self,
1824        fn_name: &'ll Value,
1825        hash: &'ll Value,
1826        bitmap_bits: &'ll Value,
1827    ) {
1828        self.call_intrinsic("llvm.instrprof.mcdc.parameters", &[], &[fn_name, hash, bitmap_bits]);
1829    }
1830
1831    #[instrument(level = "debug", skip(self))]
1832    pub(crate) fn mcdc_tvbitmap_update(
1833        &mut self,
1834        fn_name: &'ll Value,
1835        hash: &'ll Value,
1836        bitmap_index: &'ll Value,
1837        mcdc_temp: &'ll Value,
1838    ) {
1839        let args = &[fn_name, hash, bitmap_index, mcdc_temp];
1840        self.call_intrinsic("llvm.instrprof.mcdc.tvbitmap.update", &[], args);
1841    }
1842
1843    #[instrument(level = "debug", skip(self))]
1844    pub(crate) fn mcdc_condbitmap_reset(&mut self, mcdc_temp: &'ll Value) {
1845        self.store(self.const_i32(0), mcdc_temp, self.tcx.data_layout.i32_align.abi);
1846    }
1847
1848    #[instrument(level = "debug", skip(self))]
1849    pub(crate) fn mcdc_condbitmap_update(&mut self, cond_index: &'ll Value, mcdc_temp: &'ll Value) {
1850        let align = self.tcx.data_layout.i32_align.abi;
1851        let current_tv_index = self.load(self.cx.type_i32(), mcdc_temp, align);
1852        let new_tv_index = self.add(current_tv_index, cond_index);
1853        self.store(new_tv_index, mcdc_temp, align);
1854    }
1855}
rustc_codegen_llvm/builder.rs

rustc_codegen_llvm/
builder.rs
