rustc_codegen_llvm/
builder.rs

1use std::borrow::{Borrow, Cow};
2use std::ops::Deref;
3use std::{iter, ptr};
4
5pub(crate) mod autodiff;
6
7use libc::{c_char, c_uint, size_t};
8use rustc_abi as abi;
9use rustc_abi::{Align, Size, WrappingRange};
10use rustc_codegen_ssa::MemFlags;
11use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind};
12use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
13use rustc_codegen_ssa::mir::place::PlaceRef;
14use rustc_codegen_ssa::traits::*;
15use rustc_data_structures::small_c_str::SmallCStr;
16use rustc_hir::def_id::DefId;
17use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
18use rustc_middle::ty::layout::{
19    FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers,
20    TyAndLayout,
21};
22use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
23use rustc_sanitizers::{cfi, kcfi};
24use rustc_session::config::OptLevel;
25use rustc_span::Span;
26use rustc_target::callconv::FnAbi;
27use rustc_target::spec::{HasTargetSpec, SanitizerSet, Target};
28use smallvec::SmallVec;
29use tracing::{debug, instrument};
30
31use crate::abi::FnAbiLlvmExt;
32use crate::attributes;
33use crate::common::Funclet;
34use crate::context::{CodegenCx, FullCx, GenericCx, SCx};
35use crate::llvm::{
36    self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, GEPNoWrapFlags, Metadata, True,
37};
38use crate::type_::Type;
39use crate::type_of::LayoutLlvmExt;
40use crate::value::Value;
41
42#[must_use]
43pub(crate) struct GenericBuilder<'a, 'll, CX: Borrow<SCx<'ll>>> {
44    pub llbuilder: &'ll mut llvm::Builder<'ll>,
45    pub cx: &'a GenericCx<'ll, CX>,
46}
47
48pub(crate) type SBuilder<'a, 'll> = GenericBuilder<'a, 'll, SCx<'ll>>;
49pub(crate) type Builder<'a, 'll, 'tcx> = GenericBuilder<'a, 'll, FullCx<'ll, 'tcx>>;
50
51impl<'a, 'll, CX: Borrow<SCx<'ll>>> Drop for GenericBuilder<'a, 'll, CX> {
52    fn drop(&mut self) {
53        unsafe {
54            llvm::LLVMDisposeBuilder(&mut *(self.llbuilder as *mut _));
55        }
56    }
57}
58
59impl<'a, 'll> SBuilder<'a, 'll> {
60    pub(crate) fn call(
61        &mut self,
62        llty: &'ll Type,
63        llfn: &'ll Value,
64        args: &[&'ll Value],
65        funclet: Option<&Funclet<'ll>>,
66    ) -> &'ll Value {
67        debug!("call {:?} with args ({:?})", llfn, args);
68
69        let args = self.check_call("call", llty, llfn, args);
70        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
71        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
72        if let Some(funclet_bundle) = funclet_bundle {
73            bundles.push(funclet_bundle);
74        }
75
76        let call = unsafe {
77            llvm::LLVMBuildCallWithOperandBundles(
78                self.llbuilder,
79                llty,
80                llfn,
81                args.as_ptr() as *const &llvm::Value,
82                args.len() as c_uint,
83                bundles.as_ptr(),
84                bundles.len() as c_uint,
85                c"".as_ptr(),
86            )
87        };
88        call
89    }
90}
91
92impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
93    fn with_cx(scx: &'a GenericCx<'ll, CX>) -> Self {
94        // Create a fresh builder from the simple context.
95        let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(scx.deref().borrow().llcx) };
96        GenericBuilder { llbuilder, cx: scx }
97    }
98
99    pub(crate) fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
100        unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) }
101    }
102
103    pub(crate) fn ret_void(&mut self) {
104        llvm::LLVMBuildRetVoid(self.llbuilder);
105    }
106
107    pub(crate) fn ret(&mut self, v: &'ll Value) {
108        unsafe {
109            llvm::LLVMBuildRet(self.llbuilder, v);
110        }
111    }
112
113    pub(crate) fn build(cx: &'a GenericCx<'ll, CX>, llbb: &'ll BasicBlock) -> Self {
114        let bx = Self::with_cx(cx);
115        unsafe {
116            llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb);
117        }
118        bx
119    }
120}
121
122/// Empty string, to be used where LLVM expects an instruction name, indicating
123/// that the instruction is to be left unnamed (i.e. numbered, in textual IR).
124// FIXME(eddyb) pass `&CStr` directly to FFI once it's a thin pointer.
125pub(crate) const UNNAMED: *const c_char = c"".as_ptr();
126
127impl<'ll, CX: Borrow<SCx<'ll>>> BackendTypes for GenericBuilder<'_, 'll, CX> {
128    type Value = <GenericCx<'ll, CX> as BackendTypes>::Value;
129    type Metadata = <GenericCx<'ll, CX> as BackendTypes>::Metadata;
130    type Function = <GenericCx<'ll, CX> as BackendTypes>::Function;
131    type BasicBlock = <GenericCx<'ll, CX> as BackendTypes>::BasicBlock;
132    type Type = <GenericCx<'ll, CX> as BackendTypes>::Type;
133    type Funclet = <GenericCx<'ll, CX> as BackendTypes>::Funclet;
134
135    type DIScope = <GenericCx<'ll, CX> as BackendTypes>::DIScope;
136    type DILocation = <GenericCx<'ll, CX> as BackendTypes>::DILocation;
137    type DIVariable = <GenericCx<'ll, CX> as BackendTypes>::DIVariable;
138}
139
140impl abi::HasDataLayout for Builder<'_, '_, '_> {
141    fn data_layout(&self) -> &abi::TargetDataLayout {
142        self.cx.data_layout()
143    }
144}
145
146impl<'tcx> ty::layout::HasTyCtxt<'tcx> for Builder<'_, '_, 'tcx> {
147    #[inline]
148    fn tcx(&self) -> TyCtxt<'tcx> {
149        self.cx.tcx
150    }
151}
152
153impl<'tcx> ty::layout::HasTypingEnv<'tcx> for Builder<'_, '_, 'tcx> {
154    fn typing_env(&self) -> ty::TypingEnv<'tcx> {
155        self.cx.typing_env()
156    }
157}
158
159impl HasTargetSpec for Builder<'_, '_, '_> {
160    #[inline]
161    fn target_spec(&self) -> &Target {
162        self.cx.target_spec()
163    }
164}
165
166impl<'tcx> LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
167    #[inline]
168    fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
169        self.cx.handle_layout_err(err, span, ty)
170    }
171}
172
173impl<'tcx> FnAbiOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
174    #[inline]
175    fn handle_fn_abi_err(
176        &self,
177        err: FnAbiError<'tcx>,
178        span: Span,
179        fn_abi_request: FnAbiRequest<'tcx>,
180    ) -> ! {
181        self.cx.handle_fn_abi_err(err, span, fn_abi_request)
182    }
183}
184
185impl<'ll, 'tcx> Deref for Builder<'_, 'll, 'tcx> {
186    type Target = CodegenCx<'ll, 'tcx>;
187
188    #[inline]
189    fn deref(&self) -> &Self::Target {
190        self.cx
191    }
192}
193
194macro_rules! math_builder_methods {
195    ($($name:ident($($arg:ident),*) => $llvm_capi:ident),+ $(,)?) => {
196        $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
197            unsafe {
198                llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED)
199            }
200        })+
201    }
202}
203
204macro_rules! set_math_builder_methods {
205    ($($name:ident($($arg:ident),*) => ($llvm_capi:ident, $llvm_set_math:ident)),+ $(,)?) => {
206        $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
207            unsafe {
208                let instr = llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED);
209                llvm::$llvm_set_math(instr);
210                instr
211            }
212        })+
213    }
214}
215
216impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
217    type CodegenCx = CodegenCx<'ll, 'tcx>;
218
219    fn build(cx: &'a CodegenCx<'ll, 'tcx>, llbb: &'ll BasicBlock) -> Self {
220        let bx = Builder::with_cx(cx);
221        unsafe {
222            llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb);
223        }
224        bx
225    }
226
227    fn cx(&self) -> &CodegenCx<'ll, 'tcx> {
228        self.cx
229    }
230
231    fn llbb(&self) -> &'ll BasicBlock {
232        unsafe { llvm::LLVMGetInsertBlock(self.llbuilder) }
233    }
234
235    fn set_span(&mut self, _span: Span) {}
236
237    fn append_block(cx: &'a CodegenCx<'ll, 'tcx>, llfn: &'ll Value, name: &str) -> &'ll BasicBlock {
238        unsafe {
239            let name = SmallCStr::new(name);
240            llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, name.as_ptr())
241        }
242    }
243
244    fn append_sibling_block(&mut self, name: &str) -> &'ll BasicBlock {
245        Self::append_block(self.cx, self.llfn(), name)
246    }
247
248    fn switch_to_block(&mut self, llbb: Self::BasicBlock) {
249        *self = Self::build(self.cx, llbb)
250    }
251
252    fn ret_void(&mut self) {
253        llvm::LLVMBuildRetVoid(self.llbuilder);
254    }
255
256    fn ret(&mut self, v: &'ll Value) {
257        unsafe {
258            llvm::LLVMBuildRet(self.llbuilder, v);
259        }
260    }
261
262    fn br(&mut self, dest: &'ll BasicBlock) {
263        unsafe {
264            llvm::LLVMBuildBr(self.llbuilder, dest);
265        }
266    }
267
268    fn cond_br(
269        &mut self,
270        cond: &'ll Value,
271        then_llbb: &'ll BasicBlock,
272        else_llbb: &'ll BasicBlock,
273    ) {
274        unsafe {
275            llvm::LLVMBuildCondBr(self.llbuilder, cond, then_llbb, else_llbb);
276        }
277    }
278
279    fn switch(
280        &mut self,
281        v: &'ll Value,
282        else_llbb: &'ll BasicBlock,
283        cases: impl ExactSizeIterator<Item = (u128, &'ll BasicBlock)>,
284    ) {
285        let switch =
286            unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) };
287        for (on_val, dest) in cases {
288            let on_val = self.const_uint_big(self.val_ty(v), on_val);
289            unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
290        }
291    }
292
293    fn switch_with_weights(
294        &mut self,
295        v: Self::Value,
296        else_llbb: Self::BasicBlock,
297        else_is_cold: bool,
298        cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock, bool)>,
299    ) {
300        if self.cx.sess().opts.optimize == rustc_session::config::OptLevel::No {
301            self.switch(v, else_llbb, cases.map(|(val, dest, _)| (val, dest)));
302            return;
303        }
304
305        let id_str = "branch_weights";
306        let id = unsafe {
307            llvm::LLVMMDStringInContext2(self.cx.llcx, id_str.as_ptr().cast(), id_str.len())
308        };
309
310        // For switch instructions with 2 targets, the `llvm.expect` intrinsic is used.
311        // This function handles switch instructions with more than 2 targets and it needs to
312        // emit branch weights metadata instead of using the intrinsic.
313        // The values 1 and 2000 are the same as the values used by the `llvm.expect` intrinsic.
314        let cold_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(1));
315        let hot_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(2000));
316        let weight =
317            |is_cold: bool| -> &Metadata { if is_cold { cold_weight } else { hot_weight } };
318
319        let mut md: SmallVec<[&Metadata; 16]> = SmallVec::with_capacity(cases.len() + 2);
320        md.push(id);
321        md.push(weight(else_is_cold));
322
323        let switch =
324            unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) };
325        for (on_val, dest, is_cold) in cases {
326            let on_val = self.const_uint_big(self.val_ty(v), on_val);
327            unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
328            md.push(weight(is_cold));
329        }
330
331        unsafe {
332            let md_node = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len() as size_t);
333            self.cx.set_metadata(switch, llvm::MD_prof, md_node);
334        }
335    }
336
337    fn invoke(
338        &mut self,
339        llty: &'ll Type,
340        fn_attrs: Option<&CodegenFnAttrs>,
341        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
342        llfn: &'ll Value,
343        args: &[&'ll Value],
344        then: &'ll BasicBlock,
345        catch: &'ll BasicBlock,
346        funclet: Option<&Funclet<'ll>>,
347        instance: Option<Instance<'tcx>>,
348    ) -> &'ll Value {
349        debug!("invoke {:?} with args ({:?})", llfn, args);
350
351        let args = self.check_call("invoke", llty, llfn, args);
352        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
353        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
354        if let Some(funclet_bundle) = funclet_bundle {
355            bundles.push(funclet_bundle);
356        }
357
358        // Emit CFI pointer type membership test
359        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
360
361        // Emit KCFI operand bundle
362        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
363        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
364            bundles.push(kcfi_bundle);
365        }
366
367        let invoke = unsafe {
368            llvm::LLVMBuildInvokeWithOperandBundles(
369                self.llbuilder,
370                llty,
371                llfn,
372                args.as_ptr(),
373                args.len() as c_uint,
374                then,
375                catch,
376                bundles.as_ptr(),
377                bundles.len() as c_uint,
378                UNNAMED,
379            )
380        };
381        if let Some(fn_abi) = fn_abi {
382            fn_abi.apply_attrs_callsite(self, invoke);
383        }
384        invoke
385    }
386
387    fn unreachable(&mut self) {
388        unsafe {
389            llvm::LLVMBuildUnreachable(self.llbuilder);
390        }
391    }
392
393    math_builder_methods! {
394        add(a, b) => LLVMBuildAdd,
395        fadd(a, b) => LLVMBuildFAdd,
396        sub(a, b) => LLVMBuildSub,
397        fsub(a, b) => LLVMBuildFSub,
398        mul(a, b) => LLVMBuildMul,
399        fmul(a, b) => LLVMBuildFMul,
400        udiv(a, b) => LLVMBuildUDiv,
401        exactudiv(a, b) => LLVMBuildExactUDiv,
402        sdiv(a, b) => LLVMBuildSDiv,
403        exactsdiv(a, b) => LLVMBuildExactSDiv,
404        fdiv(a, b) => LLVMBuildFDiv,
405        urem(a, b) => LLVMBuildURem,
406        srem(a, b) => LLVMBuildSRem,
407        frem(a, b) => LLVMBuildFRem,
408        shl(a, b) => LLVMBuildShl,
409        lshr(a, b) => LLVMBuildLShr,
410        ashr(a, b) => LLVMBuildAShr,
411        and(a, b) => LLVMBuildAnd,
412        or(a, b) => LLVMBuildOr,
413        xor(a, b) => LLVMBuildXor,
414        neg(x) => LLVMBuildNeg,
415        fneg(x) => LLVMBuildFNeg,
416        not(x) => LLVMBuildNot,
417        unchecked_sadd(x, y) => LLVMBuildNSWAdd,
418        unchecked_uadd(x, y) => LLVMBuildNUWAdd,
419        unchecked_ssub(x, y) => LLVMBuildNSWSub,
420        unchecked_usub(x, y) => LLVMBuildNUWSub,
421        unchecked_smul(x, y) => LLVMBuildNSWMul,
422        unchecked_umul(x, y) => LLVMBuildNUWMul,
423    }
424
425    fn unchecked_suadd(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
426        unsafe {
427            let add = llvm::LLVMBuildAdd(self.llbuilder, a, b, UNNAMED);
428            if llvm::LLVMIsAInstruction(add).is_some() {
429                llvm::LLVMSetNUW(add, True);
430                llvm::LLVMSetNSW(add, True);
431            }
432            add
433        }
434    }
435    fn unchecked_susub(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
436        unsafe {
437            let sub = llvm::LLVMBuildSub(self.llbuilder, a, b, UNNAMED);
438            if llvm::LLVMIsAInstruction(sub).is_some() {
439                llvm::LLVMSetNUW(sub, True);
440                llvm::LLVMSetNSW(sub, True);
441            }
442            sub
443        }
444    }
445    fn unchecked_sumul(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
446        unsafe {
447            let mul = llvm::LLVMBuildMul(self.llbuilder, a, b, UNNAMED);
448            if llvm::LLVMIsAInstruction(mul).is_some() {
449                llvm::LLVMSetNUW(mul, True);
450                llvm::LLVMSetNSW(mul, True);
451            }
452            mul
453        }
454    }
455
456    fn or_disjoint(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
457        unsafe {
458            let or = llvm::LLVMBuildOr(self.llbuilder, a, b, UNNAMED);
459
460            // If a and b are both values, then `or` is a value, rather than
461            // an instruction, so we need to check before setting the flag.
462            // (See also `LLVMBuildNUWNeg` which also needs a check.)
463            if llvm::LLVMIsAInstruction(or).is_some() {
464                llvm::LLVMSetIsDisjoint(or, True);
465            }
466            or
467        }
468    }
469
470    set_math_builder_methods! {
471        fadd_fast(x, y) => (LLVMBuildFAdd, LLVMRustSetFastMath),
472        fsub_fast(x, y) => (LLVMBuildFSub, LLVMRustSetFastMath),
473        fmul_fast(x, y) => (LLVMBuildFMul, LLVMRustSetFastMath),
474        fdiv_fast(x, y) => (LLVMBuildFDiv, LLVMRustSetFastMath),
475        frem_fast(x, y) => (LLVMBuildFRem, LLVMRustSetFastMath),
476        fadd_algebraic(x, y) => (LLVMBuildFAdd, LLVMRustSetAlgebraicMath),
477        fsub_algebraic(x, y) => (LLVMBuildFSub, LLVMRustSetAlgebraicMath),
478        fmul_algebraic(x, y) => (LLVMBuildFMul, LLVMRustSetAlgebraicMath),
479        fdiv_algebraic(x, y) => (LLVMBuildFDiv, LLVMRustSetAlgebraicMath),
480        frem_algebraic(x, y) => (LLVMBuildFRem, LLVMRustSetAlgebraicMath),
481    }
482
483    fn checked_binop(
484        &mut self,
485        oop: OverflowOp,
486        ty: Ty<'tcx>,
487        lhs: Self::Value,
488        rhs: Self::Value,
489    ) -> (Self::Value, Self::Value) {
490        let (size, signed) = ty.int_size_and_signed(self.tcx);
491        let width = size.bits();
492
493        if oop == OverflowOp::Sub && !signed {
494            // Emit sub and icmp instead of llvm.usub.with.overflow. LLVM considers these
495            // to be the canonical form. It will attempt to reform llvm.usub.with.overflow
496            // in the backend if profitable.
497            let sub = self.sub(lhs, rhs);
498            let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs);
499            return (sub, cmp);
500        }
501
502        let oop_str = match oop {
503            OverflowOp::Add => "add",
504            OverflowOp::Sub => "sub",
505            OverflowOp::Mul => "mul",
506        };
507
508        let name = format!("llvm.{}{oop_str}.with.overflow", if signed { 's' } else { 'u' });
509
510        let res = self.call_intrinsic(name, &[self.type_ix(width)], &[lhs, rhs]);
511        (self.extract_value(res, 0), self.extract_value(res, 1))
512    }
513
514    fn from_immediate(&mut self, val: Self::Value) -> Self::Value {
515        if self.cx().val_ty(val) == self.cx().type_i1() {
516            self.zext(val, self.cx().type_i8())
517        } else {
518            val
519        }
520    }
521
522    fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value {
523        if scalar.is_bool() {
524            return self.unchecked_utrunc(val, self.cx().type_i1());
525        }
526        val
527    }
528
529    fn alloca(&mut self, size: Size, align: Align) -> &'ll Value {
530        let mut bx = Builder::with_cx(self.cx);
531        bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
532        let ty = self.cx().type_array(self.cx().type_i8(), size.bytes());
533        unsafe {
534            let alloca = llvm::LLVMBuildAlloca(bx.llbuilder, ty, UNNAMED);
535            llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
536            // Cast to default addrspace if necessary
537            llvm::LLVMBuildPointerCast(bx.llbuilder, alloca, self.cx().type_ptr(), UNNAMED)
538        }
539    }
540
541    fn dynamic_alloca(&mut self, size: &'ll Value, align: Align) -> &'ll Value {
542        unsafe {
543            let alloca =
544                llvm::LLVMBuildArrayAlloca(self.llbuilder, self.cx().type_i8(), size, UNNAMED);
545            llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
546            // Cast to default addrspace if necessary
547            llvm::LLVMBuildPointerCast(self.llbuilder, alloca, self.cx().type_ptr(), UNNAMED)
548        }
549    }
550
551    fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
552        unsafe {
553            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
554            let align = align.min(self.cx().tcx.sess.target.max_reliable_alignment());
555            llvm::LLVMSetAlignment(load, align.bytes() as c_uint);
556            load
557        }
558    }
559
560    fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value {
561        unsafe {
562            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
563            llvm::LLVMSetVolatile(load, llvm::True);
564            load
565        }
566    }
567
568    fn atomic_load(
569        &mut self,
570        ty: &'ll Type,
571        ptr: &'ll Value,
572        order: rustc_middle::ty::AtomicOrdering,
573        size: Size,
574    ) -> &'ll Value {
575        unsafe {
576            let load = llvm::LLVMRustBuildAtomicLoad(
577                self.llbuilder,
578                ty,
579                ptr,
580                UNNAMED,
581                AtomicOrdering::from_generic(order),
582            );
583            // LLVM requires the alignment of atomic loads to be at least the size of the type.
584            llvm::LLVMSetAlignment(load, size.bytes() as c_uint);
585            load
586        }
587    }
588
589    #[instrument(level = "trace", skip(self))]
590    fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> {
591        if place.layout.is_unsized() {
592            let tail = self.tcx.struct_tail_for_codegen(place.layout.ty, self.typing_env());
593            if matches!(tail.kind(), ty::Foreign(..)) {
594                // Unsized locals and, at least conceptually, even unsized arguments must be copied
595                // around, which requires dynamically determining their size. Therefore, we cannot
596                // allow `extern` types here. Consult t-opsem before removing this check.
597                panic!("unsized locals must not be `extern` types");
598            }
599        }
600        assert_eq!(place.val.llextra.is_some(), place.layout.is_unsized());
601
602        if place.layout.is_zst() {
603            return OperandRef::zero_sized(place.layout);
604        }
605
606        #[instrument(level = "trace", skip(bx))]
607        fn scalar_load_metadata<'a, 'll, 'tcx>(
608            bx: &mut Builder<'a, 'll, 'tcx>,
609            load: &'ll Value,
610            scalar: abi::Scalar,
611            layout: TyAndLayout<'tcx>,
612            offset: Size,
613        ) {
614            if bx.cx.sess().opts.optimize == OptLevel::No {
615                // Don't emit metadata we're not going to use
616                return;
617            }
618
619            if !scalar.is_uninit_valid() {
620                bx.noundef_metadata(load);
621            }
622
623            match scalar.primitive() {
624                abi::Primitive::Int(..) => {
625                    if !scalar.is_always_valid(bx) {
626                        bx.range_metadata(load, scalar.valid_range(bx));
627                    }
628                }
629                abi::Primitive::Pointer(_) => {
630                    if !scalar.valid_range(bx).contains(0) {
631                        bx.nonnull_metadata(load);
632                    }
633
634                    if let Some(pointee) = layout.pointee_info_at(bx, offset) {
635                        if let Some(_) = pointee.safe {
636                            bx.align_metadata(load, pointee.align);
637                        }
638                    }
639                }
640                abi::Primitive::Float(_) => {}
641            }
642        }
643
644        let val = if let Some(_) = place.val.llextra {
645            // FIXME: Merge with the `else` below?
646            OperandValue::Ref(place.val)
647        } else if place.layout.is_llvm_immediate() {
648            let mut const_llval = None;
649            let llty = place.layout.llvm_type(self);
650            unsafe {
651                if let Some(global) = llvm::LLVMIsAGlobalVariable(place.val.llval) {
652                    if llvm::LLVMIsGlobalConstant(global) == llvm::True {
653                        if let Some(init) = llvm::LLVMGetInitializer(global) {
654                            if self.val_ty(init) == llty {
655                                const_llval = Some(init);
656                            }
657                        }
658                    }
659                }
660            }
661            let llval = const_llval.unwrap_or_else(|| {
662                let load = self.load(llty, place.val.llval, place.val.align);
663                if let abi::BackendRepr::Scalar(scalar) = place.layout.backend_repr {
664                    scalar_load_metadata(self, load, scalar, place.layout, Size::ZERO);
665                    self.to_immediate_scalar(load, scalar)
666                } else {
667                    load
668                }
669            });
670            OperandValue::Immediate(llval)
671        } else if let abi::BackendRepr::ScalarPair(a, b) = place.layout.backend_repr {
672            let b_offset = a.size(self).align_to(b.align(self).abi);
673
674            let mut load = |i, scalar: abi::Scalar, layout, align, offset| {
675                let llptr = if i == 0 {
676                    place.val.llval
677                } else {
678                    self.inbounds_ptradd(place.val.llval, self.const_usize(b_offset.bytes()))
679                };
680                let llty = place.layout.scalar_pair_element_llvm_type(self, i, false);
681                let load = self.load(llty, llptr, align);
682                scalar_load_metadata(self, load, scalar, layout, offset);
683                self.to_immediate_scalar(load, scalar)
684            };
685
686            OperandValue::Pair(
687                load(0, a, place.layout, place.val.align, Size::ZERO),
688                load(1, b, place.layout, place.val.align.restrict_for_offset(b_offset), b_offset),
689            )
690        } else {
691            OperandValue::Ref(place.val)
692        };
693
694        OperandRef { val, layout: place.layout }
695    }
696
697    fn write_operand_repeatedly(
698        &mut self,
699        cg_elem: OperandRef<'tcx, &'ll Value>,
700        count: u64,
701        dest: PlaceRef<'tcx, &'ll Value>,
702    ) {
703        let zero = self.const_usize(0);
704        let count = self.const_usize(count);
705
706        let header_bb = self.append_sibling_block("repeat_loop_header");
707        let body_bb = self.append_sibling_block("repeat_loop_body");
708        let next_bb = self.append_sibling_block("repeat_loop_next");
709
710        self.br(header_bb);
711
712        let mut header_bx = Self::build(self.cx, header_bb);
713        let i = header_bx.phi(self.val_ty(zero), &[zero], &[self.llbb()]);
714
715        let keep_going = header_bx.icmp(IntPredicate::IntULT, i, count);
716        header_bx.cond_br(keep_going, body_bb, next_bb);
717
718        let mut body_bx = Self::build(self.cx, body_bb);
719        let dest_elem = dest.project_index(&mut body_bx, i);
720        cg_elem.val.store(&mut body_bx, dest_elem);
721
722        let next = body_bx.unchecked_uadd(i, self.const_usize(1));
723        body_bx.br(header_bb);
724        header_bx.add_incoming_to_phi(i, next, body_bb);
725
726        *self = Self::build(self.cx, next_bb);
727    }
728
729    fn range_metadata(&mut self, load: &'ll Value, range: WrappingRange) {
730        if self.cx.sess().opts.optimize == OptLevel::No {
731            // Don't emit metadata we're not going to use
732            return;
733        }
734
735        unsafe {
736            let llty = self.cx.val_ty(load);
737            let md = [
738                llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.start)),
739                llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.end.wrapping_add(1))),
740            ];
741            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len());
742            self.set_metadata(load, llvm::MD_range, md);
743        }
744    }
745
746    fn nonnull_metadata(&mut self, load: &'ll Value) {
747        unsafe {
748            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
749            self.set_metadata(load, llvm::MD_nonnull, md);
750        }
751    }
752
753    fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value {
754        self.store_with_flags(val, ptr, align, MemFlags::empty())
755    }
756
757    fn store_with_flags(
758        &mut self,
759        val: &'ll Value,
760        ptr: &'ll Value,
761        align: Align,
762        flags: MemFlags,
763    ) -> &'ll Value {
764        debug!("Store {:?} -> {:?} ({:?})", val, ptr, flags);
765        assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
766        unsafe {
767            let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
768            let align = align.min(self.cx().tcx.sess.target.max_reliable_alignment());
769            let align =
770                if flags.contains(MemFlags::UNALIGNED) { 1 } else { align.bytes() as c_uint };
771            llvm::LLVMSetAlignment(store, align);
772            if flags.contains(MemFlags::VOLATILE) {
773                llvm::LLVMSetVolatile(store, llvm::True);
774            }
775            if flags.contains(MemFlags::NONTEMPORAL) {
776                // Make sure that the current target architectures supports "sane" non-temporal
777                // stores, i.e., non-temporal stores that are equivalent to regular stores except
778                // for performance. LLVM doesn't seem to care about this, and will happily treat
779                // `!nontemporal` stores as-if they were normal stores (for reordering optimizations
780                // etc) even on x86, despite later lowering them to MOVNT which do *not* behave like
781                // regular stores but require special fences. So we keep a list of architectures
782                // where `!nontemporal` is known to be truly just a hint, and use regular stores
783                // everywhere else. (In the future, we could alternatively ensure that an sfence
784                // gets emitted after a sequence of movnt before any kind of synchronizing
785                // operation. But it's not clear how to do that with LLVM.)
786                // For more context, see <https://github.com/rust-lang/rust/issues/114582> and
787                // <https://github.com/llvm/llvm-project/issues/64521>.
788                const WELL_BEHAVED_NONTEMPORAL_ARCHS: &[&str] =
789                    &["aarch64", "arm", "riscv32", "riscv64"];
790
791                let use_nontemporal =
792                    WELL_BEHAVED_NONTEMPORAL_ARCHS.contains(&&*self.cx.tcx.sess.target.arch);
793                if use_nontemporal {
794                    // According to LLVM [1] building a nontemporal store must
795                    // *always* point to a metadata value of the integer 1.
796                    //
797                    // [1]: https://llvm.org/docs/LangRef.html#store-instruction
798                    let one = llvm::LLVMValueAsMetadata(self.cx.const_i32(1));
799                    let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, &one, 1);
800                    self.set_metadata(store, llvm::MD_nontemporal, md);
801                }
802            }
803            store
804        }
805    }
806
807    fn atomic_store(
808        &mut self,
809        val: &'ll Value,
810        ptr: &'ll Value,
811        order: rustc_middle::ty::AtomicOrdering,
812        size: Size,
813    ) {
814        debug!("Store {:?} -> {:?}", val, ptr);
815        assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
816        unsafe {
817            let store = llvm::LLVMRustBuildAtomicStore(
818                self.llbuilder,
819                val,
820                ptr,
821                AtomicOrdering::from_generic(order),
822            );
823            // LLVM requires the alignment of atomic stores to be at least the size of the type.
824            llvm::LLVMSetAlignment(store, size.bytes() as c_uint);
825        }
826    }
827
828    fn gep(&mut self, ty: &'ll Type, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value {
829        unsafe {
830            llvm::LLVMBuildGEPWithNoWrapFlags(
831                self.llbuilder,
832                ty,
833                ptr,
834                indices.as_ptr(),
835                indices.len() as c_uint,
836                UNNAMED,
837                GEPNoWrapFlags::default(),
838            )
839        }
840    }
841
842    fn inbounds_gep(
843        &mut self,
844        ty: &'ll Type,
845        ptr: &'ll Value,
846        indices: &[&'ll Value],
847    ) -> &'ll Value {
848        unsafe {
849            llvm::LLVMBuildGEPWithNoWrapFlags(
850                self.llbuilder,
851                ty,
852                ptr,
853                indices.as_ptr(),
854                indices.len() as c_uint,
855                UNNAMED,
856                GEPNoWrapFlags::InBounds,
857            )
858        }
859    }
860
861    fn inbounds_nuw_gep(
862        &mut self,
863        ty: &'ll Type,
864        ptr: &'ll Value,
865        indices: &[&'ll Value],
866    ) -> &'ll Value {
867        unsafe {
868            llvm::LLVMBuildGEPWithNoWrapFlags(
869                self.llbuilder,
870                ty,
871                ptr,
872                indices.as_ptr(),
873                indices.len() as c_uint,
874                UNNAMED,
875                GEPNoWrapFlags::InBounds | GEPNoWrapFlags::NUW,
876            )
877        }
878    }
879
880    /* Casts */
881    fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
882        unsafe { llvm::LLVMBuildTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
883    }
884
885    fn unchecked_utrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
886        debug_assert_ne!(self.val_ty(val), dest_ty);
887
888        let trunc = self.trunc(val, dest_ty);
889        unsafe {
890            if llvm::LLVMIsAInstruction(trunc).is_some() {
891                llvm::LLVMSetNUW(trunc, True);
892            }
893        }
894        trunc
895    }
896
897    fn unchecked_strunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
898        debug_assert_ne!(self.val_ty(val), dest_ty);
899
900        let trunc = self.trunc(val, dest_ty);
901        unsafe {
902            if llvm::LLVMIsAInstruction(trunc).is_some() {
903                llvm::LLVMSetNSW(trunc, True);
904            }
905        }
906        trunc
907    }
908
909    fn sext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
910        unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, UNNAMED) }
911    }
912
913    fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
914        self.call_intrinsic("llvm.fptoui.sat", &[dest_ty, self.val_ty(val)], &[val])
915    }
916
917    fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
918        self.call_intrinsic("llvm.fptosi.sat", &[dest_ty, self.val_ty(val)], &[val])
919    }
920
921    fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
922        // On WebAssembly the `fptoui` and `fptosi` instructions currently have
923        // poor codegen. The reason for this is that the corresponding wasm
924        // instructions, `i32.trunc_f32_s` for example, will trap when the float
925        // is out-of-bounds, infinity, or nan. This means that LLVM
926        // automatically inserts control flow around `fptoui` and `fptosi`
927        // because the LLVM instruction `fptoui` is defined as producing a
928        // poison value, not having UB on out-of-bounds values.
929        //
930        // This method, however, is only used with non-saturating casts that
931        // have UB on out-of-bounds values. This means that it's ok if we use
932        // the raw wasm instruction since out-of-bounds values can do whatever
933        // we like. To ensure that LLVM picks the right instruction we choose
934        // the raw wasm intrinsic functions which avoid LLVM inserting all the
935        // other control flow automatically.
936        if self.sess().target.is_like_wasm {
937            let src_ty = self.cx.val_ty(val);
938            if self.cx.type_kind(src_ty) != TypeKind::Vector {
939                let float_width = self.cx.float_width(src_ty);
940                let int_width = self.cx.int_width(dest_ty);
941                if matches!((int_width, float_width), (32 | 64, 32 | 64)) {
942                    return self.call_intrinsic(
943                        "llvm.wasm.trunc.unsigned",
944                        &[dest_ty, src_ty],
945                        &[val],
946                    );
947                }
948            }
949        }
950        unsafe { llvm::LLVMBuildFPToUI(self.llbuilder, val, dest_ty, UNNAMED) }
951    }
952
953    fn fptosi(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
954        // see `fptoui` above for why wasm is different here
955        if self.sess().target.is_like_wasm {
956            let src_ty = self.cx.val_ty(val);
957            if self.cx.type_kind(src_ty) != TypeKind::Vector {
958                let float_width = self.cx.float_width(src_ty);
959                let int_width = self.cx.int_width(dest_ty);
960                if matches!((int_width, float_width), (32 | 64, 32 | 64)) {
961                    return self.call_intrinsic(
962                        "llvm.wasm.trunc.signed",
963                        &[dest_ty, src_ty],
964                        &[val],
965                    );
966                }
967            }
968        }
969        unsafe { llvm::LLVMBuildFPToSI(self.llbuilder, val, dest_ty, UNNAMED) }
970    }
971
972    fn uitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
973        unsafe { llvm::LLVMBuildUIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
974    }
975
976    fn sitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
977        unsafe { llvm::LLVMBuildSIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
978    }
979
980    fn fptrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
981        unsafe { llvm::LLVMBuildFPTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
982    }
983
984    fn fpext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
985        unsafe { llvm::LLVMBuildFPExt(self.llbuilder, val, dest_ty, UNNAMED) }
986    }
987
988    fn ptrtoint(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
989        unsafe { llvm::LLVMBuildPtrToInt(self.llbuilder, val, dest_ty, UNNAMED) }
990    }
991
992    fn inttoptr(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
993        unsafe { llvm::LLVMBuildIntToPtr(self.llbuilder, val, dest_ty, UNNAMED) }
994    }
995
996    fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
997        unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) }
998    }
999
1000    fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value {
1001        unsafe {
1002            llvm::LLVMBuildIntCast2(
1003                self.llbuilder,
1004                val,
1005                dest_ty,
1006                if is_signed { True } else { False },
1007                UNNAMED,
1008            )
1009        }
1010    }
1011
1012    fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1013        unsafe { llvm::LLVMBuildPointerCast(self.llbuilder, val, dest_ty, UNNAMED) }
1014    }
1015
1016    /* Comparisons */
1017    fn icmp(&mut self, op: IntPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1018        let op = llvm::IntPredicate::from_generic(op);
1019        unsafe { llvm::LLVMBuildICmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
1020    }
1021
1022    fn fcmp(&mut self, op: RealPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1023        let op = llvm::RealPredicate::from_generic(op);
1024        unsafe { llvm::LLVMBuildFCmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
1025    }
1026
1027    fn three_way_compare(
1028        &mut self,
1029        ty: Ty<'tcx>,
1030        lhs: Self::Value,
1031        rhs: Self::Value,
1032    ) -> Option<Self::Value> {
1033        // FIXME: See comment on the definition of `three_way_compare`.
1034        if crate::llvm_util::get_version() < (20, 0, 0) {
1035            return None;
1036        }
1037
1038        let size = ty.primitive_size(self.tcx);
1039        let name = if ty.is_signed() { "llvm.scmp" } else { "llvm.ucmp" };
1040
1041        Some(self.call_intrinsic(name, &[self.type_i8(), self.type_ix(size.bits())], &[lhs, rhs]))
1042    }
1043
1044    /* Miscellaneous instructions */
1045    fn memcpy(
1046        &mut self,
1047        dst: &'ll Value,
1048        dst_align: Align,
1049        src: &'ll Value,
1050        src_align: Align,
1051        size: &'ll Value,
1052        flags: MemFlags,
1053    ) {
1054        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported");
1055        let size = self.intcast(size, self.type_isize(), false);
1056        let is_volatile = flags.contains(MemFlags::VOLATILE);
1057        unsafe {
1058            llvm::LLVMRustBuildMemCpy(
1059                self.llbuilder,
1060                dst,
1061                dst_align.bytes() as c_uint,
1062                src,
1063                src_align.bytes() as c_uint,
1064                size,
1065                is_volatile,
1066            );
1067        }
1068    }
1069
1070    fn memmove(
1071        &mut self,
1072        dst: &'ll Value,
1073        dst_align: Align,
1074        src: &'ll Value,
1075        src_align: Align,
1076        size: &'ll Value,
1077        flags: MemFlags,
1078    ) {
1079        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memmove not supported");
1080        let size = self.intcast(size, self.type_isize(), false);
1081        let is_volatile = flags.contains(MemFlags::VOLATILE);
1082        unsafe {
1083            llvm::LLVMRustBuildMemMove(
1084                self.llbuilder,
1085                dst,
1086                dst_align.bytes() as c_uint,
1087                src,
1088                src_align.bytes() as c_uint,
1089                size,
1090                is_volatile,
1091            );
1092        }
1093    }
1094
1095    fn memset(
1096        &mut self,
1097        ptr: &'ll Value,
1098        fill_byte: &'ll Value,
1099        size: &'ll Value,
1100        align: Align,
1101        flags: MemFlags,
1102    ) {
1103        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memset not supported");
1104        let is_volatile = flags.contains(MemFlags::VOLATILE);
1105        unsafe {
1106            llvm::LLVMRustBuildMemSet(
1107                self.llbuilder,
1108                ptr,
1109                align.bytes() as c_uint,
1110                fill_byte,
1111                size,
1112                is_volatile,
1113            );
1114        }
1115    }
1116
1117    fn select(
1118        &mut self,
1119        cond: &'ll Value,
1120        then_val: &'ll Value,
1121        else_val: &'ll Value,
1122    ) -> &'ll Value {
1123        unsafe { llvm::LLVMBuildSelect(self.llbuilder, cond, then_val, else_val, UNNAMED) }
1124    }
1125
1126    fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1127        unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
1128    }
1129
1130    fn extract_element(&mut self, vec: &'ll Value, idx: &'ll Value) -> &'ll Value {
1131        unsafe { llvm::LLVMBuildExtractElement(self.llbuilder, vec, idx, UNNAMED) }
1132    }
1133
1134    fn vector_splat(&mut self, num_elts: usize, elt: &'ll Value) -> &'ll Value {
1135        unsafe {
1136            let elt_ty = self.cx.val_ty(elt);
1137            let undef = llvm::LLVMGetUndef(self.type_vector(elt_ty, num_elts as u64));
1138            let vec = self.insert_element(undef, elt, self.cx.const_i32(0));
1139            let vec_i32_ty = self.type_vector(self.type_i32(), num_elts as u64);
1140            self.shuffle_vector(vec, undef, self.const_null(vec_i32_ty))
1141        }
1142    }
1143
1144    fn extract_value(&mut self, agg_val: &'ll Value, idx: u64) -> &'ll Value {
1145        assert_eq!(idx as c_uint as u64, idx);
1146        unsafe { llvm::LLVMBuildExtractValue(self.llbuilder, agg_val, idx as c_uint, UNNAMED) }
1147    }
1148
1149    fn insert_value(&mut self, agg_val: &'ll Value, elt: &'ll Value, idx: u64) -> &'ll Value {
1150        assert_eq!(idx as c_uint as u64, idx);
1151        unsafe { llvm::LLVMBuildInsertValue(self.llbuilder, agg_val, elt, idx as c_uint, UNNAMED) }
1152    }
1153
1154    fn set_personality_fn(&mut self, personality: &'ll Value) {
1155        unsafe {
1156            llvm::LLVMSetPersonalityFn(self.llfn(), personality);
1157        }
1158    }
1159
1160    fn cleanup_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) {
1161        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1162        let landing_pad = self.landing_pad(ty, pers_fn, 0);
1163        unsafe {
1164            llvm::LLVMSetCleanup(landing_pad, llvm::True);
1165        }
1166        (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1))
1167    }
1168
1169    fn filter_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) {
1170        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1171        let landing_pad = self.landing_pad(ty, pers_fn, 1);
1172        self.add_clause(landing_pad, self.const_array(self.type_ptr(), &[]));
1173        (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1))
1174    }
1175
1176    fn resume(&mut self, exn0: &'ll Value, exn1: &'ll Value) {
1177        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1178        let mut exn = self.const_poison(ty);
1179        exn = self.insert_value(exn, exn0, 0);
1180        exn = self.insert_value(exn, exn1, 1);
1181        unsafe {
1182            llvm::LLVMBuildResume(self.llbuilder, exn);
1183        }
1184    }
1185
1186    fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> {
1187        let ret = unsafe {
1188            llvm::LLVMBuildCleanupPad(
1189                self.llbuilder,
1190                parent,
1191                args.as_ptr(),
1192                args.len() as c_uint,
1193                c"cleanuppad".as_ptr(),
1194            )
1195        };
1196        Funclet::new(ret.expect("LLVM does not have support for cleanuppad"))
1197    }
1198
1199    fn cleanup_ret(&mut self, funclet: &Funclet<'ll>, unwind: Option<&'ll BasicBlock>) {
1200        unsafe {
1201            llvm::LLVMBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind)
1202                .expect("LLVM does not have support for cleanupret");
1203        }
1204    }
1205
1206    fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> {
1207        let ret = unsafe {
1208            llvm::LLVMBuildCatchPad(
1209                self.llbuilder,
1210                parent,
1211                args.as_ptr(),
1212                args.len() as c_uint,
1213                c"catchpad".as_ptr(),
1214            )
1215        };
1216        Funclet::new(ret.expect("LLVM does not have support for catchpad"))
1217    }
1218
1219    fn catch_switch(
1220        &mut self,
1221        parent: Option<&'ll Value>,
1222        unwind: Option<&'ll BasicBlock>,
1223        handlers: &[&'ll BasicBlock],
1224    ) -> &'ll Value {
1225        let ret = unsafe {
1226            llvm::LLVMBuildCatchSwitch(
1227                self.llbuilder,
1228                parent,
1229                unwind,
1230                handlers.len() as c_uint,
1231                c"catchswitch".as_ptr(),
1232            )
1233        };
1234        let ret = ret.expect("LLVM does not have support for catchswitch");
1235        for handler in handlers {
1236            unsafe {
1237                llvm::LLVMAddHandler(ret, handler);
1238            }
1239        }
1240        ret
1241    }
1242
1243    // Atomic Operations
1244    fn atomic_cmpxchg(
1245        &mut self,
1246        dst: &'ll Value,
1247        cmp: &'ll Value,
1248        src: &'ll Value,
1249        order: rustc_middle::ty::AtomicOrdering,
1250        failure_order: rustc_middle::ty::AtomicOrdering,
1251        weak: bool,
1252    ) -> (&'ll Value, &'ll Value) {
1253        let weak = if weak { llvm::True } else { llvm::False };
1254        unsafe {
1255            let value = llvm::LLVMBuildAtomicCmpXchg(
1256                self.llbuilder,
1257                dst,
1258                cmp,
1259                src,
1260                AtomicOrdering::from_generic(order),
1261                AtomicOrdering::from_generic(failure_order),
1262                llvm::False, // SingleThreaded
1263            );
1264            llvm::LLVMSetWeak(value, weak);
1265            let val = self.extract_value(value, 0);
1266            let success = self.extract_value(value, 1);
1267            (val, success)
1268        }
1269    }
1270
1271    fn atomic_rmw(
1272        &mut self,
1273        op: rustc_codegen_ssa::common::AtomicRmwBinOp,
1274        dst: &'ll Value,
1275        mut src: &'ll Value,
1276        order: rustc_middle::ty::AtomicOrdering,
1277    ) -> &'ll Value {
1278        // The only RMW operation that LLVM supports on pointers is compare-exchange.
1279        let requires_cast_to_int = self.val_ty(src) == self.type_ptr()
1280            && op != rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicXchg;
1281        if requires_cast_to_int {
1282            src = self.ptrtoint(src, self.type_isize());
1283        }
1284        let mut res = unsafe {
1285            llvm::LLVMBuildAtomicRMW(
1286                self.llbuilder,
1287                AtomicRmwBinOp::from_generic(op),
1288                dst,
1289                src,
1290                AtomicOrdering::from_generic(order),
1291                llvm::False, // SingleThreaded
1292            )
1293        };
1294        if requires_cast_to_int {
1295            res = self.inttoptr(res, self.type_ptr());
1296        }
1297        res
1298    }
1299
1300    fn atomic_fence(
1301        &mut self,
1302        order: rustc_middle::ty::AtomicOrdering,
1303        scope: SynchronizationScope,
1304    ) {
1305        let single_threaded = match scope {
1306            SynchronizationScope::SingleThread => llvm::True,
1307            SynchronizationScope::CrossThread => llvm::False,
1308        };
1309        unsafe {
1310            llvm::LLVMBuildFence(
1311                self.llbuilder,
1312                AtomicOrdering::from_generic(order),
1313                single_threaded,
1314                UNNAMED,
1315            );
1316        }
1317    }
1318
1319    fn set_invariant_load(&mut self, load: &'ll Value) {
1320        unsafe {
1321            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
1322            self.set_metadata(load, llvm::MD_invariant_load, md);
1323        }
1324    }
1325
1326    fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) {
1327        self.call_lifetime_intrinsic("llvm.lifetime.start", ptr, size);
1328    }
1329
1330    fn lifetime_end(&mut self, ptr: &'ll Value, size: Size) {
1331        self.call_lifetime_intrinsic("llvm.lifetime.end", ptr, size);
1332    }
1333
1334    fn call(
1335        &mut self,
1336        llty: &'ll Type,
1337        fn_attrs: Option<&CodegenFnAttrs>,
1338        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1339        llfn: &'ll Value,
1340        args: &[&'ll Value],
1341        funclet: Option<&Funclet<'ll>>,
1342        instance: Option<Instance<'tcx>>,
1343    ) -> &'ll Value {
1344        debug!("call {:?} with args ({:?})", llfn, args);
1345
1346        let args = self.check_call("call", llty, llfn, args);
1347        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1348        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1349        if let Some(funclet_bundle) = funclet_bundle {
1350            bundles.push(funclet_bundle);
1351        }
1352
1353        // Emit CFI pointer type membership test
1354        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
1355
1356        // Emit KCFI operand bundle
1357        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
1358        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
1359            bundles.push(kcfi_bundle);
1360        }
1361
1362        let call = unsafe {
1363            llvm::LLVMBuildCallWithOperandBundles(
1364                self.llbuilder,
1365                llty,
1366                llfn,
1367                args.as_ptr() as *const &llvm::Value,
1368                args.len() as c_uint,
1369                bundles.as_ptr(),
1370                bundles.len() as c_uint,
1371                c"".as_ptr(),
1372            )
1373        };
1374        if let Some(fn_abi) = fn_abi {
1375            fn_abi.apply_attrs_callsite(self, call);
1376        }
1377        call
1378    }
1379
1380    fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1381        unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) }
1382    }
1383
1384    fn apply_attrs_to_cleanup_callsite(&mut self, llret: &'ll Value) {
1385        // Cleanup is always the cold path.
1386        let cold_inline = llvm::AttributeKind::Cold.create_attr(self.llcx);
1387        attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_inline]);
1388    }
1389}
1390
1391impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> {
1392    fn get_static(&mut self, def_id: DefId) -> &'ll Value {
1393        // Forward to the `get_static` method of `CodegenCx`
1394        let global = self.cx().get_static(def_id);
1395        if self.cx().tcx.is_thread_local_static(def_id) {
1396            let pointer =
1397                self.call_intrinsic("llvm.threadlocal.address", &[self.val_ty(global)], &[global]);
1398            // Cast to default address space if globals are in a different addrspace
1399            self.pointercast(pointer, self.type_ptr())
1400        } else {
1401            // Cast to default address space if globals are in a different addrspace
1402            self.cx().const_pointercast(global, self.type_ptr())
1403        }
1404    }
1405}
1406
1407impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1408    pub(crate) fn llfn(&self) -> &'ll Value {
1409        unsafe { llvm::LLVMGetBasicBlockParent(self.llbb()) }
1410    }
1411}
1412
1413impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1414    fn position_at_start(&mut self, llbb: &'ll BasicBlock) {
1415        unsafe {
1416            llvm::LLVMRustPositionBuilderAtStart(self.llbuilder, llbb);
1417        }
1418    }
1419}
1420impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1421    fn align_metadata(&mut self, load: &'ll Value, align: Align) {
1422        unsafe {
1423            let md = [llvm::LLVMValueAsMetadata(self.cx.const_u64(align.bytes()))];
1424            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len());
1425            self.set_metadata(load, llvm::MD_align, md);
1426        }
1427    }
1428
1429    fn noundef_metadata(&mut self, load: &'ll Value) {
1430        unsafe {
1431            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
1432            self.set_metadata(load, llvm::MD_noundef, md);
1433        }
1434    }
1435
1436    pub(crate) fn set_unpredictable(&mut self, inst: &'ll Value) {
1437        unsafe {
1438            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
1439            self.set_metadata(inst, llvm::MD_unpredictable, md);
1440        }
1441    }
1442}
1443impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1444    pub(crate) fn minnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1445        unsafe { llvm::LLVMRustBuildMinNum(self.llbuilder, lhs, rhs) }
1446    }
1447
1448    pub(crate) fn maxnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1449        unsafe { llvm::LLVMRustBuildMaxNum(self.llbuilder, lhs, rhs) }
1450    }
1451
1452    pub(crate) fn insert_element(
1453        &mut self,
1454        vec: &'ll Value,
1455        elt: &'ll Value,
1456        idx: &'ll Value,
1457    ) -> &'ll Value {
1458        unsafe { llvm::LLVMBuildInsertElement(self.llbuilder, vec, elt, idx, UNNAMED) }
1459    }
1460
1461    pub(crate) fn shuffle_vector(
1462        &mut self,
1463        v1: &'ll Value,
1464        v2: &'ll Value,
1465        mask: &'ll Value,
1466    ) -> &'ll Value {
1467        unsafe { llvm::LLVMBuildShuffleVector(self.llbuilder, v1, v2, mask, UNNAMED) }
1468    }
1469
1470    pub(crate) fn vector_reduce_fadd(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1471        unsafe { llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src) }
1472    }
1473    pub(crate) fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1474        unsafe { llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src) }
1475    }
1476    pub(crate) fn vector_reduce_fadd_reassoc(
1477        &mut self,
1478        acc: &'ll Value,
1479        src: &'ll Value,
1480    ) -> &'ll Value {
1481        unsafe {
1482            let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src);
1483            llvm::LLVMRustSetAllowReassoc(instr);
1484            instr
1485        }
1486    }
1487    pub(crate) fn vector_reduce_fmul_reassoc(
1488        &mut self,
1489        acc: &'ll Value,
1490        src: &'ll Value,
1491    ) -> &'ll Value {
1492        unsafe {
1493            let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src);
1494            llvm::LLVMRustSetAllowReassoc(instr);
1495            instr
1496        }
1497    }
1498    pub(crate) fn vector_reduce_add(&mut self, src: &'ll Value) -> &'ll Value {
1499        unsafe { llvm::LLVMRustBuildVectorReduceAdd(self.llbuilder, src) }
1500    }
1501    pub(crate) fn vector_reduce_mul(&mut self, src: &'ll Value) -> &'ll Value {
1502        unsafe { llvm::LLVMRustBuildVectorReduceMul(self.llbuilder, src) }
1503    }
1504    pub(crate) fn vector_reduce_and(&mut self, src: &'ll Value) -> &'ll Value {
1505        unsafe { llvm::LLVMRustBuildVectorReduceAnd(self.llbuilder, src) }
1506    }
1507    pub(crate) fn vector_reduce_or(&mut self, src: &'ll Value) -> &'ll Value {
1508        unsafe { llvm::LLVMRustBuildVectorReduceOr(self.llbuilder, src) }
1509    }
1510    pub(crate) fn vector_reduce_xor(&mut self, src: &'ll Value) -> &'ll Value {
1511        unsafe { llvm::LLVMRustBuildVectorReduceXor(self.llbuilder, src) }
1512    }
1513    pub(crate) fn vector_reduce_fmin(&mut self, src: &'ll Value) -> &'ll Value {
1514        unsafe {
1515            llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ false)
1516        }
1517    }
1518    pub(crate) fn vector_reduce_fmax(&mut self, src: &'ll Value) -> &'ll Value {
1519        unsafe {
1520            llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ false)
1521        }
1522    }
1523    pub(crate) fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1524        unsafe { llvm::LLVMRustBuildVectorReduceMin(self.llbuilder, src, is_signed) }
1525    }
1526    pub(crate) fn vector_reduce_max(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1527        unsafe { llvm::LLVMRustBuildVectorReduceMax(self.llbuilder, src, is_signed) }
1528    }
1529
1530    pub(crate) fn add_clause(&mut self, landing_pad: &'ll Value, clause: &'ll Value) {
1531        unsafe {
1532            llvm::LLVMAddClause(landing_pad, clause);
1533        }
1534    }
1535
1536    pub(crate) fn catch_ret(
1537        &mut self,
1538        funclet: &Funclet<'ll>,
1539        unwind: &'ll BasicBlock,
1540    ) -> &'ll Value {
1541        let ret = unsafe { llvm::LLVMBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) };
1542        ret.expect("LLVM does not have support for catchret")
1543    }
1544
1545    fn check_call<'b>(
1546        &mut self,
1547        typ: &str,
1548        fn_ty: &'ll Type,
1549        llfn: &'ll Value,
1550        args: &'b [&'ll Value],
1551    ) -> Cow<'b, [&'ll Value]> {
1552        assert!(
1553            self.cx.type_kind(fn_ty) == TypeKind::Function,
1554            "builder::{typ} not passed a function, but {fn_ty:?}"
1555        );
1556
1557        let param_tys = self.cx.func_params_types(fn_ty);
1558
1559        let all_args_match = iter::zip(&param_tys, args.iter().map(|&v| self.cx.val_ty(v)))
1560            .all(|(expected_ty, actual_ty)| *expected_ty == actual_ty);
1561
1562        if all_args_match {
1563            return Cow::Borrowed(args);
1564        }
1565
1566        let casted_args: Vec<_> = iter::zip(param_tys, args)
1567            .enumerate()
1568            .map(|(i, (expected_ty, &actual_val))| {
1569                let actual_ty = self.cx.val_ty(actual_val);
1570                if expected_ty != actual_ty {
1571                    debug!(
1572                        "type mismatch in function call of {:?}. \
1573                            Expected {:?} for param {}, got {:?}; injecting bitcast",
1574                        llfn, expected_ty, i, actual_ty
1575                    );
1576                    self.bitcast(actual_val, expected_ty)
1577                } else {
1578                    actual_val
1579                }
1580            })
1581            .collect();
1582
1583        Cow::Owned(casted_args)
1584    }
1585
1586    pub(crate) fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1587        unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
1588    }
1589}
1590
1591impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1592    pub(crate) fn call_intrinsic(
1593        &mut self,
1594        base_name: impl Into<Cow<'static, str>>,
1595        type_params: &[&'ll Type],
1596        args: &[&'ll Value],
1597    ) -> &'ll Value {
1598        let (ty, f) = self.cx.get_intrinsic(base_name.into(), type_params);
1599        self.call(ty, None, None, f, args, None, None)
1600    }
1601
1602    fn call_lifetime_intrinsic(&mut self, intrinsic: &'static str, ptr: &'ll Value, size: Size) {
1603        let size = size.bytes();
1604        if size == 0 {
1605            return;
1606        }
1607
1608        if !self.cx().sess().emit_lifetime_markers() {
1609            return;
1610        }
1611
1612        self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[self.cx.const_u64(size), ptr]);
1613    }
1614}
1615impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1616    pub(crate) fn phi(
1617        &mut self,
1618        ty: &'ll Type,
1619        vals: &[&'ll Value],
1620        bbs: &[&'ll BasicBlock],
1621    ) -> &'ll Value {
1622        assert_eq!(vals.len(), bbs.len());
1623        let phi = unsafe { llvm::LLVMBuildPhi(self.llbuilder, ty, UNNAMED) };
1624        unsafe {
1625            llvm::LLVMAddIncoming(phi, vals.as_ptr(), bbs.as_ptr(), vals.len() as c_uint);
1626            phi
1627        }
1628    }
1629
1630    fn add_incoming_to_phi(&mut self, phi: &'ll Value, val: &'ll Value, bb: &'ll BasicBlock) {
1631        unsafe {
1632            llvm::LLVMAddIncoming(phi, &val, &bb, 1 as c_uint);
1633        }
1634    }
1635}
1636impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1637    pub(crate) fn landing_pad(
1638        &mut self,
1639        ty: &'ll Type,
1640        pers_fn: &'ll Value,
1641        num_clauses: usize,
1642    ) -> &'ll Value {
1643        // Use LLVMSetPersonalityFn to set the personality. It supports arbitrary Consts while,
1644        // LLVMBuildLandingPad requires the argument to be a Function (as of LLVM 12). The
1645        // personality lives on the parent function anyway.
1646        self.set_personality_fn(pers_fn);
1647        unsafe {
1648            llvm::LLVMBuildLandingPad(self.llbuilder, ty, None, num_clauses as c_uint, UNNAMED)
1649        }
1650    }
1651
1652    pub(crate) fn callbr(
1653        &mut self,
1654        llty: &'ll Type,
1655        fn_attrs: Option<&CodegenFnAttrs>,
1656        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1657        llfn: &'ll Value,
1658        args: &[&'ll Value],
1659        default_dest: &'ll BasicBlock,
1660        indirect_dest: &[&'ll BasicBlock],
1661        funclet: Option<&Funclet<'ll>>,
1662        instance: Option<Instance<'tcx>>,
1663    ) -> &'ll Value {
1664        debug!("invoke {:?} with args ({:?})", llfn, args);
1665
1666        let args = self.check_call("callbr", llty, llfn, args);
1667        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1668        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1669        if let Some(funclet_bundle) = funclet_bundle {
1670            bundles.push(funclet_bundle);
1671        }
1672
1673        // Emit CFI pointer type membership test
1674        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
1675
1676        // Emit KCFI operand bundle
1677        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
1678        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
1679            bundles.push(kcfi_bundle);
1680        }
1681
1682        let callbr = unsafe {
1683            llvm::LLVMBuildCallBr(
1684                self.llbuilder,
1685                llty,
1686                llfn,
1687                default_dest,
1688                indirect_dest.as_ptr(),
1689                indirect_dest.len() as c_uint,
1690                args.as_ptr(),
1691                args.len() as c_uint,
1692                bundles.as_ptr(),
1693                bundles.len() as c_uint,
1694                UNNAMED,
1695            )
1696        };
1697        if let Some(fn_abi) = fn_abi {
1698            fn_abi.apply_attrs_callsite(self, callbr);
1699        }
1700        callbr
1701    }
1702
1703    // Emits CFI pointer type membership tests.
1704    fn cfi_type_test(
1705        &mut self,
1706        fn_attrs: Option<&CodegenFnAttrs>,
1707        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1708        instance: Option<Instance<'tcx>>,
1709        llfn: &'ll Value,
1710    ) {
1711        let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
1712        if self.tcx.sess.is_sanitizer_cfi_enabled()
1713            && let Some(fn_abi) = fn_abi
1714            && is_indirect_call
1715        {
1716            if let Some(fn_attrs) = fn_attrs
1717                && fn_attrs.no_sanitize.contains(SanitizerSet::CFI)
1718            {
1719                return;
1720            }
1721
1722            let mut options = cfi::TypeIdOptions::empty();
1723            if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
1724                options.insert(cfi::TypeIdOptions::GENERALIZE_POINTERS);
1725            }
1726            if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
1727                options.insert(cfi::TypeIdOptions::NORMALIZE_INTEGERS);
1728            }
1729
1730            let typeid = if let Some(instance) = instance {
1731                cfi::typeid_for_instance(self.tcx, instance, options)
1732            } else {
1733                cfi::typeid_for_fnabi(self.tcx, fn_abi, options)
1734            };
1735            let typeid_metadata = self.cx.typeid_metadata(typeid).unwrap();
1736            let dbg_loc = self.get_dbg_loc();
1737
1738            // Test whether the function pointer is associated with the type identifier using the
1739            // llvm.type.test intrinsic. The LowerTypeTests link-time optimization pass replaces
1740            // calls to this intrinsic with code to test type membership.
1741            let typeid = self.get_metadata_value(typeid_metadata);
1742            let cond = self.call_intrinsic("llvm.type.test", &[], &[llfn, typeid]);
1743            let bb_pass = self.append_sibling_block("type_test.pass");
1744            let bb_fail = self.append_sibling_block("type_test.fail");
1745            self.cond_br(cond, bb_pass, bb_fail);
1746
1747            self.switch_to_block(bb_fail);
1748            if let Some(dbg_loc) = dbg_loc {
1749                self.set_dbg_loc(dbg_loc);
1750            }
1751            self.abort();
1752            self.unreachable();
1753
1754            self.switch_to_block(bb_pass);
1755            if let Some(dbg_loc) = dbg_loc {
1756                self.set_dbg_loc(dbg_loc);
1757            }
1758        }
1759    }
1760
1761    // Emits KCFI operand bundles.
1762    fn kcfi_operand_bundle(
1763        &mut self,
1764        fn_attrs: Option<&CodegenFnAttrs>,
1765        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1766        instance: Option<Instance<'tcx>>,
1767        llfn: &'ll Value,
1768    ) -> Option<llvm::OperandBundleBox<'ll>> {
1769        let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
1770        let kcfi_bundle = if self.tcx.sess.is_sanitizer_kcfi_enabled()
1771            && let Some(fn_abi) = fn_abi
1772            && is_indirect_call
1773        {
1774            if let Some(fn_attrs) = fn_attrs
1775                && fn_attrs.no_sanitize.contains(SanitizerSet::KCFI)
1776            {
1777                return None;
1778            }
1779
1780            let mut options = kcfi::TypeIdOptions::empty();
1781            if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
1782                options.insert(kcfi::TypeIdOptions::GENERALIZE_POINTERS);
1783            }
1784            if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
1785                options.insert(kcfi::TypeIdOptions::NORMALIZE_INTEGERS);
1786            }
1787
1788            let kcfi_typeid = if let Some(instance) = instance {
1789                kcfi::typeid_for_instance(self.tcx, instance, options)
1790            } else {
1791                kcfi::typeid_for_fnabi(self.tcx, fn_abi, options)
1792            };
1793
1794            Some(llvm::OperandBundleBox::new("kcfi", &[self.const_u32(kcfi_typeid)]))
1795        } else {
1796            None
1797        };
1798        kcfi_bundle
1799    }
1800
1801    /// Emits a call to `llvm.instrprof.increment`. Used by coverage instrumentation.
1802    #[instrument(level = "debug", skip(self))]
1803    pub(crate) fn instrprof_increment(
1804        &mut self,
1805        fn_name: &'ll Value,
1806        hash: &'ll Value,
1807        num_counters: &'ll Value,
1808        index: &'ll Value,
1809    ) {
1810        self.call_intrinsic("llvm.instrprof.increment", &[], &[fn_name, hash, num_counters, index]);
1811    }
1812
1813    /// Emits a call to `llvm.instrprof.mcdc.parameters`.
1814    ///
1815    /// This doesn't produce any code directly, but is used as input by
1816    /// the LLVM pass that handles coverage instrumentation.
1817    ///
1818    /// (See clang's [`CodeGenPGO::emitMCDCParameters`] for comparison.)
1819    ///
1820    /// [`CodeGenPGO::emitMCDCParameters`]:
1821    ///     https://github.com/rust-lang/llvm-project/blob/5399a24/clang/lib/CodeGen/CodeGenPGO.cpp#L1124
1822    #[instrument(level = "debug", skip(self))]
1823    pub(crate) fn mcdc_parameters(
1824        &mut self,
1825        fn_name: &'ll Value,
1826        hash: &'ll Value,
1827        bitmap_bits: &'ll Value,
1828    ) {
1829        self.call_intrinsic("llvm.instrprof.mcdc.parameters", &[], &[fn_name, hash, bitmap_bits]);
1830    }
1831
1832    #[instrument(level = "debug", skip(self))]
1833    pub(crate) fn mcdc_tvbitmap_update(
1834        &mut self,
1835        fn_name: &'ll Value,
1836        hash: &'ll Value,
1837        bitmap_index: &'ll Value,
1838        mcdc_temp: &'ll Value,
1839    ) {
1840        let args = &[fn_name, hash, bitmap_index, mcdc_temp];
1841        self.call_intrinsic("llvm.instrprof.mcdc.tvbitmap.update", &[], args);
1842    }
1843
1844    #[instrument(level = "debug", skip(self))]
1845    pub(crate) fn mcdc_condbitmap_reset(&mut self, mcdc_temp: &'ll Value) {
1846        self.store(self.const_i32(0), mcdc_temp, self.tcx.data_layout.i32_align.abi);
1847    }
1848
1849    #[instrument(level = "debug", skip(self))]
1850    pub(crate) fn mcdc_condbitmap_update(&mut self, cond_index: &'ll Value, mcdc_temp: &'ll Value) {
1851        let align = self.tcx.data_layout.i32_align.abi;
1852        let current_tv_index = self.load(self.cx.type_i32(), mcdc_temp, align);
1853        let new_tv_index = self.add(current_tv_index, cond_index);
1854        self.store(new_tv_index, mcdc_temp, align);
1855    }
1856}
rustc_codegen_llvm/builder.rs

rustc_codegen_llvm/
builder.rs
