; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=aarch64 < %s -verify-machineinstrs | FileCheck %s
; RUN: llc -mtriple=aarch64 < %s -verify-machineinstrs -global-isel -global-isel-abort=2 | FileCheck %s
; Dynamically-sized allocation, needs a loop which can handle any size at
; runtime. The final iteration of the loop will temporarily put SP below the
; target address, but this doesn't break any of the ABI constraints on the
; stack, and also doesn't probe below the target SP value.
define void @dynamic(i64 %size, ptr %out) #0 {
; CHECK-LABEL: dynamic:
; CHECK: // %bb.0:
; CHECK-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
; CHECK-NEXT: .cfi_def_cfa_offset 16
; CHECK-NEXT: mov x29, sp
; CHECK-NEXT: .cfi_def_cfa w29, 16
; CHECK-NEXT: .cfi_offset w30, -8
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: add x9, x0, #15
; CHECK-NEXT: mov x8, sp
; CHECK-NEXT: and x9, x9, #0xfffffffffffffff0
; CHECK-NEXT: sub x8, x8, x9
; CHECK-NEXT: .LBB0_1: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: sub sp, sp, #1, lsl #12 // =4096
; CHECK-NEXT: cmp sp, x8
; CHECK-NEXT: b.le .LBB0_3
; CHECK-NEXT: // %bb.2: // in Loop: Header=BB0_1 Depth=1
; CHECK-NEXT: str xzr, [sp]
; CHECK-NEXT: b .LBB0_1
; CHECK-NEXT: .LBB0_3:
; CHECK-NEXT: mov sp, x8
; CHECK-NEXT: ldr xzr, [sp]
; CHECK-NEXT: str x8, [x1]
; CHECK-NEXT: mov sp, x29
; CHECK-NEXT: .cfi_def_cfa wsp, 16
; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
; CHECK-NEXT: .cfi_def_cfa_offset 0
; CHECK-NEXT: .cfi_restore w30
; CHECK-NEXT: .cfi_restore w29
; CHECK-NEXT: ret
%v = alloca i8, i64 %size, align 1
store ptr %v, ptr %out, align 8
ret void
}
; This function has a fixed-size stack slot and a dynamic one. The fixed size
; slot isn't large enough that we would normally probe it, but we need to do so
; here otherwise the gap between the CSR save and the first probe of the
; dynamic allocation could be too far apart when the size of the dynamic
; allocation is close to the guard size.
define void @dynamic_fixed(i64 %size, ptr %out1, ptr %out2) #0 {
; CHECK-LABEL: dynamic_fixed:
; CHECK: // %bb.0:
; CHECK-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
; CHECK-NEXT: .cfi_def_cfa_offset 16
; CHECK-NEXT: mov x29, sp
; CHECK-NEXT: .cfi_def_cfa w29, 16
; CHECK-NEXT: .cfi_offset w30, -8
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: str xzr, [sp, #-64]!
; CHECK-NEXT: add x9, x0, #15
; CHECK-NEXT: mov x8, sp
; CHECK-DAG: sub x10, x29, #64
; CHECK-DAG: and x9, x9, #0xfffffffffffffff0
; CHECK-NOT: INVALID_TO_BREAK_UP_CHECK_DAG
; CHECK-DAG: str x10, [x1]
; CHECK-DAG: sub x8, x8, x9
; CHECK-NEXT: .LBB1_1: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: sub sp, sp, #1, lsl #12 // =4096
; CHECK-NEXT: cmp sp, x8
; CHECK-NEXT: b.le .LBB1_3
; CHECK-NEXT: // %bb.2: // in Loop: Header=BB1_1 Depth=1
; CHECK-NEXT: str xzr, [sp]
; CHECK-NEXT: b .LBB1_1
; CHECK-NEXT: .LBB1_3:
; CHECK-NEXT: mov sp, x8
; CHECK-NEXT: ldr xzr, [sp]
; CHECK-NEXT: str x8, [x2]
; CHECK-NEXT: mov sp, x29
; CHECK-NEXT: .cfi_def_cfa wsp, 16
; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
; CHECK-NEXT: .cfi_def_cfa_offset 0
; CHECK-NEXT: .cfi_restore w30
; CHECK-NEXT: .cfi_restore w29
; CHECK-NEXT: ret
%v1 = alloca i8, i64 64, align 1
store ptr %v1, ptr %out1, align 8
%v2 = alloca i8, i64 %size, align 1
store ptr %v2, ptr %out2, align 8
ret void
}
; Dynamic allocation, with an alignment requirement greater than the alignment
; of SP. Done by ANDing the target SP with a constant to align it down, then
; doing the loop as normal. Note that we also re-align the stack in the prolog,
; which isn't actually needed because the only aligned allocations are dynamic,
; this is done even without stack probing.
define void @dynamic_align_64(i64 %size, ptr %out) #0 {
; CHECK-LABEL: dynamic_align_64:
; CHECK: // %bb.0:
; CHECK-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
; CHECK-NEXT: .cfi_def_cfa_offset 32
; CHECK-NEXT: str x19, [sp, #16] // 8-byte Folded Spill
; CHECK-NEXT: mov x29, sp
; CHECK-NEXT: .cfi_def_cfa w29, 32
; CHECK-NEXT: .cfi_offset w19, -16
; CHECK-NEXT: .cfi_offset w30, -24
; CHECK-NEXT: .cfi_offset w29, -32
; CHECK-NEXT: sub x9, sp, #32
; CHECK-NEXT: and sp, x9, #0xffffffffffffffc0
; CHECK-NEXT: add x9, x0, #15
; CHECK-NEXT: mov x8, sp
; CHECK-DAG: str xzr, [sp]
; CHECK-DAG: and x9, x9, #0xfffffffffffffff0
; CHECK-NOT: INVALID_TO_BREAK_UP_CHECK_DAG
; CHECK-DAG: mov x19, sp
; CHECK-DAG: sub x8, x8, x9
; CHECK-NEXT: and x8, x8, #0xffffffffffffffc0
; CHECK-NEXT: .LBB2_1: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: sub sp, sp, #1, lsl #12 // =4096
; CHECK-NEXT: cmp sp, x8
; CHECK-NEXT: b.le .LBB2_3
; CHECK-NEXT: // %bb.2: // in Loop: Header=BB2_1 Depth=1
; CHECK-NEXT: str xzr, [sp]
; CHECK-NEXT: b .LBB2_1
; CHECK-NEXT: .LBB2_3:
; CHECK-NEXT: mov sp, x8
; CHECK-NEXT: ldr xzr, [sp]
; CHECK-NEXT: str x8, [x1]
; CHECK-NEXT: mov sp, x29
; CHECK-NEXT: .cfi_def_cfa wsp, 32
; CHECK-NEXT: ldr x19, [sp, #16] // 8-byte Folded Reload
; CHECK-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload
; CHECK-NEXT: .cfi_def_cfa_offset 0
; CHECK-NEXT: .cfi_restore w19
; CHECK-NEXT: .cfi_restore w30
; CHECK-NEXT: .cfi_restore w29
; CHECK-NEXT: ret
%v = alloca i8, i64 %size, align 64
store ptr %v, ptr %out, align 8
ret void
}
; Dynamic allocation, with an alignment greater than the stack guard size. The
; only difference to the dynamic allocation is the constant used for aligning
; the target SP, the loop will probe the whole allocation without needing to
; know about the alignment padding.
define void @dynamic_align_8192(i64 %size, ptr %out) #0 {
; CHECK-LABEL: dynamic_align_8192:
; CHECK: // %bb.0:
; CHECK-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
; CHECK-NEXT: .cfi_def_cfa_offset 32
; CHECK-NEXT: str x19, [sp, #16] // 8-byte Folded Spill
; CHECK-NEXT: mov x29, sp
; CHECK-NEXT: .cfi_def_cfa w29, 32
; CHECK-NEXT: .cfi_offset w19, -16
; CHECK-NEXT: .cfi_offset w30, -24
; CHECK-NEXT: .cfi_offset w29, -32
; CHECK-NEXT: sub x9, sp, #1, lsl #12 // =4096
; CHECK-NEXT: sub x9, x9, #4064
; CHECK-NEXT: and x9, x9, #0xffffffffffffe000
; CHECK-NEXT: .LBB3_1: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: sub sp, sp, #1, lsl #12 // =4096
; CHECK-NEXT: cmp sp, x9
; CHECK-NEXT: b.le .LBB3_3
; CHECK-NEXT: // %bb.2: // in Loop: Header=BB3_1 Depth=1
; CHECK-NEXT: str xzr, [sp]
; CHECK-NEXT: b .LBB3_1
; CHECK-NEXT: .LBB3_3:
; CHECK-NEXT: mov sp, x9
; CHECK-NEXT: add x9, x0, #15
; CHECK-NEXT: mov x8, sp
; CHECK-DAG: ldr xzr, [sp]
; CHECK-DAG: and x9, x9, #0xfffffffffffffff0
; CHECK-NOT: INVALID_TO_BREAK_UP_CHECK_DAG
; CHECK-DAG: mov x19, sp
; CHECK-DAG: sub x8, x8, x9
; CHECK-NEXT: and x8, x8, #0xffffffffffffe000
; CHECK-NEXT: .LBB3_4: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: sub sp, sp, #1, lsl #12 // =4096
; CHECK-NEXT: cmp sp, x8
; CHECK-NEXT: b.le .LBB3_6
; CHECK-NEXT: // %bb.5: // in Loop: Header=BB3_4 Depth=1
; CHECK-NEXT: str xzr, [sp]
; CHECK-NEXT: b .LBB3_4
; CHECK-NEXT: .LBB3_6:
; CHECK-NEXT: mov sp, x8
; CHECK-NEXT: ldr xzr, [sp]
; CHECK-NEXT: str x8, [x1]
; CHECK-NEXT: mov sp, x29
; CHECK-NEXT: .cfi_def_cfa wsp, 32
; CHECK-NEXT: ldr x19, [sp, #16] // 8-byte Folded Reload
; CHECK-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload
; CHECK-NEXT: .cfi_def_cfa_offset 0
; CHECK-NEXT: .cfi_restore w19
; CHECK-NEXT: .cfi_restore w30
; CHECK-NEXT: .cfi_restore w29
; CHECK-NEXT: ret
%v = alloca i8, i64 %size, align 8192
store ptr %v, ptr %out, align 8
ret void
}
; For 64k guard pages, the only difference is the constant subtracted from SP
; in the loop.
define void @dynamic_64k_guard(i64 %size, ptr %out) #0 "stack-probe-size"="65536" {
; CHECK-LABEL: dynamic_64k_guard:
; CHECK: // %bb.0:
; CHECK-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
; CHECK-NEXT: .cfi_def_cfa_offset 16
; CHECK-NEXT: mov x29, sp
; CHECK-NEXT: .cfi_def_cfa w29, 16
; CHECK-NEXT: .cfi_offset w30, -8
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: add x9, x0, #15
; CHECK-NEXT: mov x8, sp
; CHECK-NEXT: and x9, x9, #0xfffffffffffffff0
; CHECK-NEXT: sub x8, x8, x9
; CHECK-NEXT: .LBB4_1: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: sub sp, sp, #16, lsl #12 // =65536
; CHECK-NEXT: cmp sp, x8
; CHECK-NEXT: b.le .LBB4_3
; CHECK-NEXT: // %bb.2: // in Loop: Header=BB4_1 Depth=1
; CHECK-NEXT: str xzr, [sp]
; CHECK-NEXT: b .LBB4_1
; CHECK-NEXT: .LBB4_3:
; CHECK-NEXT: mov sp, x8
; CHECK-NEXT: ldr xzr, [sp]
; CHECK-NEXT: str x8, [x1]
; CHECK-NEXT: mov sp, x29
; CHECK-NEXT: .cfi_def_cfa wsp, 16
; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
; CHECK-NEXT: .cfi_def_cfa_offset 0
; CHECK-NEXT: .cfi_restore w30
; CHECK-NEXT: .cfi_restore w29
; CHECK-NEXT: ret
%v = alloca i8, i64 %size, align 1
store ptr %v, ptr %out, align 8
ret void
}
; If a function has variable-sized stack objects, then any function calls which
; need to pass arguments on the stack must allocate the stack space for them
; dynamically, to ensure they are at the bottom of the frame. We need to probe
; that space when it is larger than the unprobed space allowed by the ABI (1024
; bytes), so this needs a very large number of arguments.
define void @no_reserved_call_frame(i64 %n) #0 {
; CHECK-LABEL: no_reserved_call_frame:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
; CHECK-NEXT: .cfi_def_cfa_offset 16
; CHECK-NEXT: mov x29, sp
; CHECK-NEXT: .cfi_def_cfa w29, 16
; CHECK-NEXT: .cfi_offset w30, -8
; CHECK-NEXT: .cfi_offset w29, -16
; CHECK-NEXT: lsl x9, x0, #2
; CHECK-NEXT: mov x8, sp
; CHECK-NEXT: add x9, x9, #15
; CHECK-NEXT: and x9, x9, #0xfffffffffffffff0
; CHECK-NEXT: sub x0, x8, x9
; CHECK-NEXT: .LBB5_1: // %entry
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: sub sp, sp, #1, lsl #12 // =4096
; CHECK-NEXT: cmp sp, x0
; CHECK-NEXT: b.le .LBB5_3
; CHECK-NEXT: // %bb.2: // %entry
; CHECK-NEXT: // in Loop: Header=BB5_1 Depth=1
; CHECK-NEXT: str xzr, [sp]
; CHECK-NEXT: b .LBB5_1
; CHECK-NEXT: .LBB5_3: // %entry
; CHECK-NEXT: mov sp, x0
; CHECK-NEXT: ldr xzr, [sp]
; CHECK-NEXT: sub sp, sp, #1104
; CHECK-NEXT: str xzr, [sp]
; CHECK-NEXT: bl callee_stack_args
; CHECK-NEXT: add sp, sp, #1104
; CHECK-NEXT: mov sp, x29
; CHECK-NEXT: .cfi_def_cfa wsp, 16
; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
; CHECK-NEXT: .cfi_def_cfa_offset 0
; CHECK-NEXT: .cfi_restore w30
; CHECK-NEXT: .cfi_restore w29
; CHECK-NEXT: ret
entry:
%v = alloca i32, i64 %n
call void @callee_stack_args(ptr %v, [138 x i64] undef)
ret void
}
; Same as above but without a variable-sized allocation, so the reserved call
; frame can be folded into the fixed-size allocation in the prologue.
define void @reserved_call_frame(i64 %n) #0 {
; CHECK-LABEL: reserved_call_frame:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
; CHECK-NEXT: .cfi_def_cfa_offset 32
; CHECK-NEXT: str x28, [sp, #16] // 8-byte Folded Spill
; CHECK-NEXT: mov x29, sp
; CHECK-NEXT: .cfi_def_cfa w29, 32
; CHECK-NEXT: .cfi_offset w28, -16
; CHECK-NEXT: .cfi_offset w30, -24
; CHECK-NEXT: .cfi_offset w29, -32
; CHECK-NEXT: sub sp, sp, #1504
; CHECK-NEXT: add x0, sp, #1104
; CHECK-NEXT: str xzr, [sp]
; CHECK-NEXT: bl callee_stack_args
; CHECK-NEXT: add sp, sp, #1504
; CHECK-NEXT: .cfi_def_cfa wsp, 32
; CHECK-NEXT: ldr x28, [sp, #16] // 8-byte Folded Reload
; CHECK-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload
; CHECK-NEXT: .cfi_def_cfa_offset 0
; CHECK-NEXT: .cfi_restore w28
; CHECK-NEXT: .cfi_restore w30
; CHECK-NEXT: .cfi_restore w29
; CHECK-NEXT: ret
entry:
%v = alloca i32, i64 100
call void @callee_stack_args(ptr %v, [138 x i64] undef)
ret void
}
declare void @callee_stack_args(ptr, [138 x i64])
; Dynamic allocation of SVE vectors
define void @dynamic_sve(i64 %size, ptr %out) #0 "target-features"="+sve" {
; CHECK-LABEL: dynamic_sve:
; CHECK: // %bb.0:
; CHECK-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
; CHECK-NEXT: .cfi_def_cfa_offset 32
; CHECK-NEXT: str x19, [sp, #16] // 8-byte Folded Spill
; CHECK-NEXT: mov x29, sp
; CHECK-NEXT: .cfi_def_cfa w29, 32
; CHECK-NEXT: .cfi_offset w19, -16
; CHECK-NEXT: .cfi_offset w30, -24
; CHECK-NEXT: .cfi_offset w29, -32
; CHECK-NEXT: rdvl x9, #1
; CHECK-NEXT: mov x10, #15 // =0xf
; CHECK-DAG: mov x8, sp
; CHECK-DAG: madd x9, x0, x9, x10
; CHECK-NEXT: and x9, x9, #0xfffffffffffffff0
; CHECK-NEXT: sub x8, x8, x9
; CHECK-NEXT: .LBB7_1: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: sub sp, sp, #1, lsl #12 // =4096
; CHECK-NEXT: cmp sp, x8
; CHECK-NEXT: b.le .LBB7_3
; CHECK-NEXT: // %bb.2: // in Loop: Header=BB7_1 Depth=1
; CHECK-NEXT: str xzr, [sp]
; CHECK-NEXT: b .LBB7_1
; CHECK-NEXT: .LBB7_3:
; CHECK-NEXT: mov sp, x8
; CHECK-NEXT: ldr xzr, [sp]
; CHECK-NEXT: str x8, [x1]
; CHECK-NEXT: mov sp, x29
; CHECK-NEXT: .cfi_def_cfa wsp, 32
; CHECK-NEXT: ldr x19, [sp, #16] // 8-byte Folded Reload
; CHECK-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload
; CHECK-NEXT: .cfi_def_cfa_offset 0
; CHECK-NEXT: .cfi_restore w19
; CHECK-NEXT: .cfi_restore w30
; CHECK-NEXT: .cfi_restore w29
; CHECK-NEXT: ret
%v = alloca <vscale x 4 x float>, i64 %size, align 16
store ptr %v, ptr %out, align 8
ret void
}
attributes #0 = { uwtable(async) "probe-stack"="inline-asm" "frame-pointer"="none" }
Codebase Browser
llvm