diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll index 89f97f6188747..c714aa4320850 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll @@ -1,4 +1,4 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph:" --version 2 +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 6 ; RUN: opt -mtriple=aarch64-none-linux-gnu -S -passes=loop-vectorize -mattr=+sve -force-vector-width=16 -force-vector-interleave=1 -scalable-vectorization=on -tail-folding-policy=dont-fold-tail %s 2>&1 | FileCheck %s -check-prefix=SCALAR_TAIL_FOLDING ; RUN: opt -mtriple=aarch64-none-linux-gnu -S -passes=loop-vectorize -mattr=+sve -force-vector-width=16 -force-vector-interleave=1 -scalable-vectorization=on -tail-folding-policy=must-fold-tail %s 2>&1 | FileCheck %s -check-prefix=PREDICATED_TAIL_FOLDING @@ -22,15 +22,15 @@ target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" ; } ;} define void @masked_strided1(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 zeroext %guard) #0 { -; SCALAR_TAIL_FOLDING-LABEL: define void @masked_strided1 -; SCALAR_TAIL_FOLDING-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0:[0-9]+]] { -; SCALAR_TAIL_FOLDING-NEXT: entry: +; SCALAR_TAIL_FOLDING-LABEL: define void @masked_strided1( +; SCALAR_TAIL_FOLDING-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0:[0-9]+]] { +; SCALAR_TAIL_FOLDING-NEXT: [[ENTRY:.*:]] ; SCALAR_TAIL_FOLDING-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32 ; SCALAR_TAIL_FOLDING-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() ; SCALAR_TAIL_FOLDING-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 1024, [[TMP1]] -; SCALAR_TAIL_FOLDING-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] -; SCALAR_TAIL_FOLDING: vector.ph: +; SCALAR_TAIL_FOLDING-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; SCALAR_TAIL_FOLDING: [[VECTOR_PH]]: ; SCALAR_TAIL_FOLDING-NEXT: [[TMP2:%.*]] = shl nuw i32 [[TMP0]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP2]] ; SCALAR_TAIL_FOLDING-NEXT: [[N_VEC:%.*]] = sub i32 1024, [[N_MOD_VF]] @@ -39,10 +39,10 @@ define void @masked_strided1(ptr noalias nocapture readonly %p, ptr noalias noca ; SCALAR_TAIL_FOLDING-NEXT: [[TMP3:%.*]] = call @llvm.stepvector.nxv16i32() ; SCALAR_TAIL_FOLDING-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement poison, i32 [[TMP2]], i64 0 ; SCALAR_TAIL_FOLDING-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector [[BROADCAST_SPLATINSERT1]], poison, zeroinitializer -; SCALAR_TAIL_FOLDING-NEXT: br label [[VECTOR_BODY:%.*]] -; SCALAR_TAIL_FOLDING: vector.body: -; SCALAR_TAIL_FOLDING-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; SCALAR_TAIL_FOLDING-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP3]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; SCALAR_TAIL_FOLDING-NEXT: br label %[[VECTOR_BODY:.*]] +; SCALAR_TAIL_FOLDING: [[VECTOR_BODY]]: +; SCALAR_TAIL_FOLDING-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; SCALAR_TAIL_FOLDING-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP3]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] ; SCALAR_TAIL_FOLDING-NEXT: [[TMP4:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT]] ; SCALAR_TAIL_FOLDING-NEXT: [[TMP5:%.*]] = shl i32 [[INDEX]], 1 ; SCALAR_TAIL_FOLDING-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i32 [[TMP5]] @@ -61,18 +61,18 @@ define void @masked_strided1(ptr noalias nocapture readonly %p, ptr noalias noca ; SCALAR_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP2]] ; SCALAR_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add nuw nsw [[VEC_IND]], [[BROADCAST_SPLAT2]] ; SCALAR_TAIL_FOLDING-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; SCALAR_TAIL_FOLDING-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] -; SCALAR_TAIL_FOLDING: middle.block: +; SCALAR_TAIL_FOLDING-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; SCALAR_TAIL_FOLDING: [[MIDDLE_BLOCK]]: ; SCALAR_TAIL_FOLDING-NEXT: [[CMP_N:%.*]] = icmp eq i32 1024, [[N_VEC]] -; SCALAR_TAIL_FOLDING-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] -; SCALAR_TAIL_FOLDING: scalar.ph: +; SCALAR_TAIL_FOLDING-NEXT: br i1 [[CMP_N]], [[FOR_END:label %.*]], label %[[SCALAR_PH]] +; SCALAR_TAIL_FOLDING: [[SCALAR_PH]]: ; -; PREDICATED_TAIL_FOLDING-LABEL: define void @masked_strided1 -; PREDICATED_TAIL_FOLDING-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0:[0-9]+]] { -; PREDICATED_TAIL_FOLDING-NEXT: entry: +; PREDICATED_TAIL_FOLDING-LABEL: define void @masked_strided1( +; PREDICATED_TAIL_FOLDING-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0:[0-9]+]] { +; PREDICATED_TAIL_FOLDING-NEXT: [[ENTRY:.*:]] ; PREDICATED_TAIL_FOLDING-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32 -; PREDICATED_TAIL_FOLDING-NEXT: br label [[VECTOR_PH:%.*]] -; PREDICATED_TAIL_FOLDING: vector.ph: +; PREDICATED_TAIL_FOLDING-NEXT: br label %[[VECTOR_PH:.*]] +; PREDICATED_TAIL_FOLDING: [[VECTOR_PH]]: ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4 ; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv16i1.i32(i32 0, i32 1024) @@ -81,11 +81,11 @@ define void @masked_strided1(ptr noalias nocapture readonly %p, ptr noalias noca ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP2:%.*]] = call @llvm.stepvector.nxv16i32() ; PREDICATED_TAIL_FOLDING-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement poison, i32 [[TMP1]], i64 0 ; PREDICATED_TAIL_FOLDING-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector [[BROADCAST_SPLATINSERT1]], poison, zeroinitializer -; PREDICATED_TAIL_FOLDING-NEXT: br label [[VECTOR_BODY:%.*]] -; PREDICATED_TAIL_FOLDING: vector.body: -; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ] -; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP2]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; PREDICATED_TAIL_FOLDING-NEXT: br label %[[VECTOR_BODY:.*]] +; PREDICATED_TAIL_FOLDING: [[VECTOR_BODY]]: +; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] +; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP3:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT]] ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP4:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP3]], zeroinitializer ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP5:%.*]] = shl i32 [[INDEX]], 1 @@ -107,10 +107,10 @@ define void @masked_strided1(ptr noalias nocapture readonly %p, ptr noalias noca ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP13:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP14:%.*]] = xor i1 [[TMP13]], true ; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add nuw nsw [[VEC_IND]], [[BROADCAST_SPLAT2]] -; PREDICATED_TAIL_FOLDING-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] -; PREDICATED_TAIL_FOLDING: middle.block: -; PREDICATED_TAIL_FOLDING-NEXT: br label [[FOR_END:%.*]] -; PREDICATED_TAIL_FOLDING: for.end: +; PREDICATED_TAIL_FOLDING-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; PREDICATED_TAIL_FOLDING: [[MIDDLE_BLOCK]]: +; PREDICATED_TAIL_FOLDING-NEXT: br label %[[FOR_END:.*]] +; PREDICATED_TAIL_FOLDING: [[FOR_END]]: ; PREDICATED_TAIL_FOLDING-NEXT: ret void ; entry: @@ -161,15 +161,15 @@ for.end: ; } ;} define void @masked_strided2(ptr noalias nocapture readnone %p, ptr noalias nocapture %q, i8 zeroext %guard) #0 { -; SCALAR_TAIL_FOLDING-LABEL: define void @masked_strided2 -; SCALAR_TAIL_FOLDING-SAME: (ptr noalias readnone captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] { -; SCALAR_TAIL_FOLDING-NEXT: entry: +; SCALAR_TAIL_FOLDING-LABEL: define void @masked_strided2( +; SCALAR_TAIL_FOLDING-SAME: ptr noalias readnone captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] { +; SCALAR_TAIL_FOLDING-NEXT: [[ENTRY:.*:]] ; SCALAR_TAIL_FOLDING-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32 ; SCALAR_TAIL_FOLDING-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() ; SCALAR_TAIL_FOLDING-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 1024, [[TMP1]] -; SCALAR_TAIL_FOLDING-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] -; SCALAR_TAIL_FOLDING: vector.ph: +; SCALAR_TAIL_FOLDING-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; SCALAR_TAIL_FOLDING: [[VECTOR_PH]]: ; SCALAR_TAIL_FOLDING-NEXT: [[TMP2:%.*]] = shl nuw i32 [[TMP0]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP2]] ; SCALAR_TAIL_FOLDING-NEXT: [[N_VEC:%.*]] = sub i32 1024, [[N_MOD_VF]] @@ -178,32 +178,32 @@ define void @masked_strided2(ptr noalias nocapture readnone %p, ptr noalias noca ; SCALAR_TAIL_FOLDING-NEXT: [[TMP3:%.*]] = call @llvm.stepvector.nxv16i32() ; SCALAR_TAIL_FOLDING-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement poison, i32 [[TMP2]], i64 0 ; SCALAR_TAIL_FOLDING-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector [[BROADCAST_SPLATINSERT1]], poison, zeroinitializer -; SCALAR_TAIL_FOLDING-NEXT: br label [[VECTOR_BODY:%.*]] -; SCALAR_TAIL_FOLDING: vector.body: -; SCALAR_TAIL_FOLDING-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; SCALAR_TAIL_FOLDING-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP3]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; SCALAR_TAIL_FOLDING-NEXT: br label %[[VECTOR_BODY:.*]] +; SCALAR_TAIL_FOLDING: [[VECTOR_BODY]]: +; SCALAR_TAIL_FOLDING-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; SCALAR_TAIL_FOLDING-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP3]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] ; SCALAR_TAIL_FOLDING-NEXT: [[TMP4:%.*]] = shl nuw nsw [[VEC_IND]], splat (i32 1) -; SCALAR_TAIL_FOLDING-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP4]] -; SCALAR_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( splat (i8 1), align 1 [[TMP5]], splat (i1 true)) -; SCALAR_TAIL_FOLDING-NEXT: [[TMP6:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT]] -; SCALAR_TAIL_FOLDING-NEXT: [[TMP7:%.*]] = or disjoint [[TMP4]], splat (i32 1) -; SCALAR_TAIL_FOLDING-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP7]] -; SCALAR_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( splat (i8 2), align 1 [[TMP8]], [[TMP6]]) +; SCALAR_TAIL_FOLDING-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP4]] +; SCALAR_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( splat (i8 1), align 1 [[WIDE_GEP]], splat (i1 true)) +; SCALAR_TAIL_FOLDING-NEXT: [[TMP5:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT]] +; SCALAR_TAIL_FOLDING-NEXT: [[TMP6:%.*]] = or disjoint [[TMP4]], splat (i32 1) +; SCALAR_TAIL_FOLDING-NEXT: [[WIDE_GEP3:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP6]] +; SCALAR_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( splat (i8 2), align 1 [[WIDE_GEP3]], [[TMP5]]) ; SCALAR_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP2]] ; SCALAR_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add nuw nsw [[VEC_IND]], [[BROADCAST_SPLAT2]] -; SCALAR_TAIL_FOLDING-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; SCALAR_TAIL_FOLDING-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] -; SCALAR_TAIL_FOLDING: middle.block: +; SCALAR_TAIL_FOLDING-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] +; SCALAR_TAIL_FOLDING-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; SCALAR_TAIL_FOLDING: [[MIDDLE_BLOCK]]: ; SCALAR_TAIL_FOLDING-NEXT: [[CMP_N:%.*]] = icmp eq i32 1024, [[N_VEC]] -; SCALAR_TAIL_FOLDING-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] -; SCALAR_TAIL_FOLDING: scalar.ph: +; SCALAR_TAIL_FOLDING-NEXT: br i1 [[CMP_N]], [[FOR_END:label %.*]], label %[[SCALAR_PH]] +; SCALAR_TAIL_FOLDING: [[SCALAR_PH]]: ; -; PREDICATED_TAIL_FOLDING-LABEL: define void @masked_strided2 -; PREDICATED_TAIL_FOLDING-SAME: (ptr noalias readnone captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] { -; PREDICATED_TAIL_FOLDING-NEXT: entry: +; PREDICATED_TAIL_FOLDING-LABEL: define void @masked_strided2( +; PREDICATED_TAIL_FOLDING-SAME: ptr noalias readnone captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] { +; PREDICATED_TAIL_FOLDING-NEXT: [[ENTRY:.*:]] ; PREDICATED_TAIL_FOLDING-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32 -; PREDICATED_TAIL_FOLDING-NEXT: br label [[VECTOR_PH:%.*]] -; PREDICATED_TAIL_FOLDING: vector.ph: +; PREDICATED_TAIL_FOLDING-NEXT: br label %[[VECTOR_PH:.*]] +; PREDICATED_TAIL_FOLDING: [[VECTOR_PH]]: ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4 ; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv16i1.i32(i32 0, i32 1024) @@ -212,28 +212,28 @@ define void @masked_strided2(ptr noalias nocapture readnone %p, ptr noalias noca ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP2:%.*]] = call @llvm.stepvector.nxv16i32() ; PREDICATED_TAIL_FOLDING-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement poison, i32 [[TMP1]], i64 0 ; PREDICATED_TAIL_FOLDING-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector [[BROADCAST_SPLATINSERT1]], poison, zeroinitializer -; PREDICATED_TAIL_FOLDING-NEXT: br label [[VECTOR_BODY:%.*]] -; PREDICATED_TAIL_FOLDING: vector.body: -; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ] -; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP2]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; PREDICATED_TAIL_FOLDING-NEXT: br label %[[VECTOR_BODY:.*]] +; PREDICATED_TAIL_FOLDING: [[VECTOR_BODY]]: +; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] +; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP3:%.*]] = shl nuw nsw [[VEC_IND]], splat (i32 1) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP3]] -; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( splat (i8 1), align 1 [[TMP4]], [[ACTIVE_LANE_MASK]]) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP5:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT]] -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP6:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP5]], zeroinitializer -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP7:%.*]] = or disjoint [[TMP3]], splat (i32 1) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP7]] -; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( splat (i8 2), align 1 [[TMP8]], [[TMP6]]) +; PREDICATED_TAIL_FOLDING-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP3]] +; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( splat (i8 1), align 1 [[WIDE_GEP]], [[ACTIVE_LANE_MASK]]) +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP4:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT]] +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP5:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP4]], zeroinitializer +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP6:%.*]] = or disjoint [[TMP3]], splat (i32 1) +; PREDICATED_TAIL_FOLDING-NEXT: [[WIDE_GEP3:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP6]] +; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( splat (i8 2), align 1 [[WIDE_GEP3]], [[TMP5]]) ; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP1]] ; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX_NEXT]], i32 1024) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP9:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP7:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP8:%.*]] = xor i1 [[TMP7]], true ; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add nuw nsw [[VEC_IND]], [[BROADCAST_SPLAT2]] -; PREDICATED_TAIL_FOLDING-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] -; PREDICATED_TAIL_FOLDING: middle.block: -; PREDICATED_TAIL_FOLDING-NEXT: br label [[FOR_END:%.*]] -; PREDICATED_TAIL_FOLDING: for.end: +; PREDICATED_TAIL_FOLDING-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] +; PREDICATED_TAIL_FOLDING: [[MIDDLE_BLOCK]]: +; PREDICATED_TAIL_FOLDING-NEXT: br label %[[FOR_END:.*]] +; PREDICATED_TAIL_FOLDING: [[FOR_END]]: ; PREDICATED_TAIL_FOLDING-NEXT: ret void ; entry: @@ -279,16 +279,16 @@ for.end: ; } ;} define void @masked_strided3(ptr noalias nocapture readnone %p, ptr noalias nocapture %q, i8 zeroext %guard1, i8 zeroext %guard2) #0 { -; SCALAR_TAIL_FOLDING-LABEL: define void @masked_strided3 -; SCALAR_TAIL_FOLDING-SAME: (ptr noalias readnone captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD1:%.*]], i8 zeroext [[GUARD2:%.*]]) #[[ATTR0]] { -; SCALAR_TAIL_FOLDING-NEXT: entry: +; SCALAR_TAIL_FOLDING-LABEL: define void @masked_strided3( +; SCALAR_TAIL_FOLDING-SAME: ptr noalias readnone captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD1:%.*]], i8 zeroext [[GUARD2:%.*]]) #[[ATTR0]] { +; SCALAR_TAIL_FOLDING-NEXT: [[ENTRY:.*:]] ; SCALAR_TAIL_FOLDING-NEXT: [[CONV:%.*]] = zext i8 [[GUARD1]] to i32 ; SCALAR_TAIL_FOLDING-NEXT: [[CONV3:%.*]] = zext i8 [[GUARD2]] to i32 ; SCALAR_TAIL_FOLDING-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() ; SCALAR_TAIL_FOLDING-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 1024, [[TMP1]] -; SCALAR_TAIL_FOLDING-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] -; SCALAR_TAIL_FOLDING: vector.ph: +; SCALAR_TAIL_FOLDING-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; SCALAR_TAIL_FOLDING: [[VECTOR_PH]]: ; SCALAR_TAIL_FOLDING-NEXT: [[TMP2:%.*]] = shl nuw i32 [[TMP0]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP2]] ; SCALAR_TAIL_FOLDING-NEXT: [[N_VEC:%.*]] = sub i32 1024, [[N_MOD_VF]] @@ -299,34 +299,34 @@ define void @masked_strided3(ptr noalias nocapture readnone %p, ptr noalias noca ; SCALAR_TAIL_FOLDING-NEXT: [[TMP3:%.*]] = call @llvm.stepvector.nxv16i32() ; SCALAR_TAIL_FOLDING-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement poison, i32 [[TMP2]], i64 0 ; SCALAR_TAIL_FOLDING-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector [[BROADCAST_SPLATINSERT3]], poison, zeroinitializer -; SCALAR_TAIL_FOLDING-NEXT: br label [[VECTOR_BODY:%.*]] -; SCALAR_TAIL_FOLDING: vector.body: -; SCALAR_TAIL_FOLDING-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; SCALAR_TAIL_FOLDING-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP3]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; SCALAR_TAIL_FOLDING-NEXT: br label %[[VECTOR_BODY:.*]] +; SCALAR_TAIL_FOLDING: [[VECTOR_BODY]]: +; SCALAR_TAIL_FOLDING-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; SCALAR_TAIL_FOLDING-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP3]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] ; SCALAR_TAIL_FOLDING-NEXT: [[TMP4:%.*]] = shl nuw nsw [[VEC_IND]], splat (i32 1) ; SCALAR_TAIL_FOLDING-NEXT: [[TMP5:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT]] -; SCALAR_TAIL_FOLDING-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP4]] -; SCALAR_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( splat (i8 1), align 1 [[TMP6]], [[TMP5]]) -; SCALAR_TAIL_FOLDING-NEXT: [[TMP7:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT2]] -; SCALAR_TAIL_FOLDING-NEXT: [[TMP8:%.*]] = or disjoint [[TMP4]], splat (i32 1) -; SCALAR_TAIL_FOLDING-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP8]] -; SCALAR_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( splat (i8 2), align 1 [[TMP9]], [[TMP7]]) +; SCALAR_TAIL_FOLDING-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP4]] +; SCALAR_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( splat (i8 1), align 1 [[WIDE_GEP]], [[TMP5]]) +; SCALAR_TAIL_FOLDING-NEXT: [[TMP6:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT2]] +; SCALAR_TAIL_FOLDING-NEXT: [[TMP7:%.*]] = or disjoint [[TMP4]], splat (i32 1) +; SCALAR_TAIL_FOLDING-NEXT: [[WIDE_GEP5:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP7]] +; SCALAR_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( splat (i8 2), align 1 [[WIDE_GEP5]], [[TMP6]]) ; SCALAR_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP2]] ; SCALAR_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add nuw nsw [[VEC_IND]], [[BROADCAST_SPLAT4]] -; SCALAR_TAIL_FOLDING-NEXT: [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; SCALAR_TAIL_FOLDING-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] -; SCALAR_TAIL_FOLDING: middle.block: +; SCALAR_TAIL_FOLDING-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] +; SCALAR_TAIL_FOLDING-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] +; SCALAR_TAIL_FOLDING: [[MIDDLE_BLOCK]]: ; SCALAR_TAIL_FOLDING-NEXT: [[CMP_N:%.*]] = icmp eq i32 1024, [[N_VEC]] -; SCALAR_TAIL_FOLDING-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] -; SCALAR_TAIL_FOLDING: scalar.ph: +; SCALAR_TAIL_FOLDING-NEXT: br i1 [[CMP_N]], [[FOR_END:label %.*]], label %[[SCALAR_PH]] +; SCALAR_TAIL_FOLDING: [[SCALAR_PH]]: ; -; PREDICATED_TAIL_FOLDING-LABEL: define void @masked_strided3 -; PREDICATED_TAIL_FOLDING-SAME: (ptr noalias readnone captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD1:%.*]], i8 zeroext [[GUARD2:%.*]]) #[[ATTR0]] { -; PREDICATED_TAIL_FOLDING-NEXT: entry: +; PREDICATED_TAIL_FOLDING-LABEL: define void @masked_strided3( +; PREDICATED_TAIL_FOLDING-SAME: ptr noalias readnone captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD1:%.*]], i8 zeroext [[GUARD2:%.*]]) #[[ATTR0]] { +; PREDICATED_TAIL_FOLDING-NEXT: [[ENTRY:.*:]] ; PREDICATED_TAIL_FOLDING-NEXT: [[CONV:%.*]] = zext i8 [[GUARD1]] to i32 ; PREDICATED_TAIL_FOLDING-NEXT: [[CONV3:%.*]] = zext i8 [[GUARD2]] to i32 -; PREDICATED_TAIL_FOLDING-NEXT: br label [[VECTOR_PH:%.*]] -; PREDICATED_TAIL_FOLDING: vector.ph: +; PREDICATED_TAIL_FOLDING-NEXT: br label %[[VECTOR_PH:.*]] +; PREDICATED_TAIL_FOLDING: [[VECTOR_PH]]: ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4 ; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv16i1.i32(i32 0, i32 1024) @@ -337,30 +337,30 @@ define void @masked_strided3(ptr noalias nocapture readnone %p, ptr noalias noca ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP2:%.*]] = call @llvm.stepvector.nxv16i32() ; PREDICATED_TAIL_FOLDING-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement poison, i32 [[TMP1]], i64 0 ; PREDICATED_TAIL_FOLDING-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector [[BROADCAST_SPLATINSERT3]], poison, zeroinitializer -; PREDICATED_TAIL_FOLDING-NEXT: br label [[VECTOR_BODY:%.*]] -; PREDICATED_TAIL_FOLDING: vector.body: -; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ] -; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP2]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; PREDICATED_TAIL_FOLDING-NEXT: br label %[[VECTOR_BODY:.*]] +; PREDICATED_TAIL_FOLDING: [[VECTOR_BODY]]: +; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] +; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP3:%.*]] = shl nuw nsw [[VEC_IND]], splat (i32 1) ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP4:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT]] ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP5:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP4]], zeroinitializer -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP3]] -; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( splat (i8 1), align 1 [[TMP6]], [[TMP5]]) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP7:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT2]] -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP8:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP7]], zeroinitializer -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP9:%.*]] = or disjoint [[TMP3]], splat (i32 1) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP9]] -; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( splat (i8 2), align 1 [[TMP10]], [[TMP8]]) +; PREDICATED_TAIL_FOLDING-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP3]] +; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( splat (i8 1), align 1 [[WIDE_GEP]], [[TMP5]]) +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP6:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT2]] +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP7:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP6]], zeroinitializer +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP8:%.*]] = or disjoint [[TMP3]], splat (i32 1) +; PREDICATED_TAIL_FOLDING-NEXT: [[WIDE_GEP5:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP8]] +; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( splat (i8 2), align 1 [[WIDE_GEP5]], [[TMP7]]) ; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP1]] ; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX_NEXT]], i32 1024) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP11:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP12:%.*]] = xor i1 [[TMP11]], true +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP9:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true ; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add nuw nsw [[VEC_IND]], [[BROADCAST_SPLAT4]] -; PREDICATED_TAIL_FOLDING-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] -; PREDICATED_TAIL_FOLDING: middle.block: -; PREDICATED_TAIL_FOLDING-NEXT: br label [[FOR_END:%.*]] -; PREDICATED_TAIL_FOLDING: for.end: +; PREDICATED_TAIL_FOLDING-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; PREDICATED_TAIL_FOLDING: [[MIDDLE_BLOCK]]: +; PREDICATED_TAIL_FOLDING-NEXT: br label %[[FOR_END:.*]] +; PREDICATED_TAIL_FOLDING: [[FOR_END]]: ; PREDICATED_TAIL_FOLDING-NEXT: ret void ; entry: @@ -419,15 +419,15 @@ for.end: ; } ;} define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 zeroext %guard) #0 { -; SCALAR_TAIL_FOLDING-LABEL: define void @masked_strided_factor4 -; SCALAR_TAIL_FOLDING-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] { -; SCALAR_TAIL_FOLDING-NEXT: entry: +; SCALAR_TAIL_FOLDING-LABEL: define void @masked_strided_factor4( +; SCALAR_TAIL_FOLDING-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] { +; SCALAR_TAIL_FOLDING-NEXT: [[ENTRY:.*:]] ; SCALAR_TAIL_FOLDING-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32 ; SCALAR_TAIL_FOLDING-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() ; SCALAR_TAIL_FOLDING-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 1024, [[TMP1]] -; SCALAR_TAIL_FOLDING-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] -; SCALAR_TAIL_FOLDING: vector.ph: +; SCALAR_TAIL_FOLDING-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; SCALAR_TAIL_FOLDING: [[VECTOR_PH]]: ; SCALAR_TAIL_FOLDING-NEXT: [[TMP2:%.*]] = shl nuw i32 [[TMP0]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP2]] ; SCALAR_TAIL_FOLDING-NEXT: [[N_VEC:%.*]] = sub i32 1024, [[N_MOD_VF]] @@ -436,10 +436,10 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali ; SCALAR_TAIL_FOLDING-NEXT: [[TMP3:%.*]] = call @llvm.stepvector.nxv16i32() ; SCALAR_TAIL_FOLDING-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement poison, i32 [[TMP2]], i64 0 ; SCALAR_TAIL_FOLDING-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector [[BROADCAST_SPLATINSERT1]], poison, zeroinitializer -; SCALAR_TAIL_FOLDING-NEXT: br label [[VECTOR_BODY:%.*]] -; SCALAR_TAIL_FOLDING: vector.body: -; SCALAR_TAIL_FOLDING-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; SCALAR_TAIL_FOLDING-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP3]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; SCALAR_TAIL_FOLDING-NEXT: br label %[[VECTOR_BODY:.*]] +; SCALAR_TAIL_FOLDING: [[VECTOR_BODY]]: +; SCALAR_TAIL_FOLDING-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; SCALAR_TAIL_FOLDING-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP3]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] ; SCALAR_TAIL_FOLDING-NEXT: [[TMP4:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT]] ; SCALAR_TAIL_FOLDING-NEXT: [[TMP5:%.*]] = shl i32 [[INDEX]], 2 ; SCALAR_TAIL_FOLDING-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i32 [[TMP5]] @@ -463,18 +463,18 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali ; SCALAR_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP2]] ; SCALAR_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add nuw nsw [[VEC_IND]], [[BROADCAST_SPLAT2]] ; SCALAR_TAIL_FOLDING-NEXT: [[TMP18:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; SCALAR_TAIL_FOLDING-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] -; SCALAR_TAIL_FOLDING: middle.block: +; SCALAR_TAIL_FOLDING-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] +; SCALAR_TAIL_FOLDING: [[MIDDLE_BLOCK]]: ; SCALAR_TAIL_FOLDING-NEXT: [[CMP_N:%.*]] = icmp eq i32 1024, [[N_VEC]] -; SCALAR_TAIL_FOLDING-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] -; SCALAR_TAIL_FOLDING: scalar.ph: +; SCALAR_TAIL_FOLDING-NEXT: br i1 [[CMP_N]], [[FOR_END:label %.*]], label %[[SCALAR_PH]] +; SCALAR_TAIL_FOLDING: [[SCALAR_PH]]: ; -; PREDICATED_TAIL_FOLDING-LABEL: define void @masked_strided_factor4 -; PREDICATED_TAIL_FOLDING-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] { -; PREDICATED_TAIL_FOLDING-NEXT: entry: +; PREDICATED_TAIL_FOLDING-LABEL: define void @masked_strided_factor4( +; PREDICATED_TAIL_FOLDING-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] { +; PREDICATED_TAIL_FOLDING-NEXT: [[ENTRY:.*:]] ; PREDICATED_TAIL_FOLDING-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32 -; PREDICATED_TAIL_FOLDING-NEXT: br label [[VECTOR_PH:%.*]] -; PREDICATED_TAIL_FOLDING: vector.ph: +; PREDICATED_TAIL_FOLDING-NEXT: br label %[[VECTOR_PH:.*]] +; PREDICATED_TAIL_FOLDING: [[VECTOR_PH]]: ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4 ; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv16i1.i32(i32 0, i32 1024) @@ -483,11 +483,11 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP2:%.*]] = call @llvm.stepvector.nxv16i32() ; PREDICATED_TAIL_FOLDING-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement poison, i32 [[TMP1]], i64 0 ; PREDICATED_TAIL_FOLDING-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector [[BROADCAST_SPLATINSERT1]], poison, zeroinitializer -; PREDICATED_TAIL_FOLDING-NEXT: br label [[VECTOR_BODY:%.*]] -; PREDICATED_TAIL_FOLDING: vector.body: -; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ] -; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP2]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; PREDICATED_TAIL_FOLDING-NEXT: br label %[[VECTOR_BODY:.*]] +; PREDICATED_TAIL_FOLDING: [[VECTOR_BODY]]: +; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] +; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP3:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT]] ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP4:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP3]], zeroinitializer ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP5:%.*]] = shl i32 [[INDEX]], 2 @@ -514,10 +514,10 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP18:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP19:%.*]] = xor i1 [[TMP18]], true ; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add nuw nsw [[VEC_IND]], [[BROADCAST_SPLAT2]] -; PREDICATED_TAIL_FOLDING-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] -; PREDICATED_TAIL_FOLDING: middle.block: -; PREDICATED_TAIL_FOLDING-NEXT: br label [[FOR_END:%.*]] -; PREDICATED_TAIL_FOLDING: for.end: +; PREDICATED_TAIL_FOLDING-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] +; PREDICATED_TAIL_FOLDING: [[MIDDLE_BLOCK]]: +; PREDICATED_TAIL_FOLDING-NEXT: br label %[[FOR_END:.*]] +; PREDICATED_TAIL_FOLDING: [[FOR_END]]: ; PREDICATED_TAIL_FOLDING-NEXT: ret void ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-zext-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-zext-costs.ll index b5c8c922a24d3..b917f2be30bc7 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-zext-costs.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-zext-costs.ll @@ -1,4 +1,4 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6 ; REQUIRES: asserts ; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=1 \ ; RUN: -enable-epilogue-vectorization=false -debug-only=loop-vectorize 2>%t < %s | FileCheck %s @@ -17,43 +17,43 @@ define void @zext_i8_i16(ptr noalias nocapture readonly %p, ptr noalias nocaptur ; CHECK-COST: Cost of 1 for VF vscale x 2: WIDEN-CAST ir<%conv> = zext ir<%0> to i16 ; CHECK-COST: Cost of 1 for VF vscale x 4: WIDEN-CAST ir<%conv> = zext ir<%0> to i16 ; CHECK-COST: Cost of 0 for VF vscale x 8: WIDEN-CAST ir<%conv> = zext ir<%0> to i16 -; CHECK-LABEL: define void @zext_i8_i16 -; CHECK-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[LEN:%.*]]) #[[ATTR0:[0-9]+]] { -; CHECK-NEXT: entry: +; CHECK-LABEL: define void @zext_i8_i16( +; CHECK-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[LEN:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[ENTRY:.*]]: ; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[LEN]], -1 ; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 ; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 ; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 4 ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]] -; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] -; CHECK: vector.ph: -; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP3]], 4 -; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]] +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP3]], 4 +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP5]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] -; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] -; CHECK: vector.body: -; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP7]], align 1 -; CHECK-NEXT: [[TMP8:%.*]] = zext [[WIDE_LOAD]] to -; CHECK-NEXT: [[TMP9:%.*]] = add [[TMP8]], splat (i16 2) -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i16, ptr [[Q]], i64 [[INDEX]] -; CHECK-NEXT: store [[TMP9]], ptr [[TMP10]], align 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]] -; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] -; CHECK: middle.block: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 1 +; CHECK-NEXT: [[TMP7:%.*]] = zext [[WIDE_LOAD]] to +; CHECK-NEXT: [[TMP8:%.*]] = add [[TMP7]], splat (i16 2) +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, ptr [[Q]], i64 [[INDEX]] +; CHECK-NEXT: store [[TMP8]], ptr [[TMP9]], align 2 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] +; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] -; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] -; CHECK: scalar.ph: -; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] -; CHECK-NEXT: br label [[FOR_BODY:%.*]] -; CHECK: for.body: -; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] +; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] +; CHECK-NEXT: br label %[[FOR_BODY:.*]] +; CHECK: [[FOR_BODY]]: +; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ] ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDVARS_IV]] -; CHECK-NEXT: [[TMP12:%.*]] = load i8, ptr [[ARRAYIDX]], align 1 -; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP12]] to i32 +; CHECK-NEXT: [[TMP11:%.*]] = load i8, ptr [[ARRAYIDX]], align 1 +; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP11]] to i32 ; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[CONV]], 2 ; CHECK-NEXT: [[CONV1:%.*]] = trunc i32 [[ADD]] to i16 ; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i16, ptr [[Q]], i64 [[INDVARS_IV]] @@ -61,8 +61,8 @@ define void @zext_i8_i16(ptr noalias nocapture readonly %p, ptr noalias nocaptur ; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 ; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 ; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[LEN]] -; CHECK-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] -; CHECK: exit: +; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] +; CHECK: [[EXIT]]: ; CHECK-NEXT: ret void ; entry: @@ -96,43 +96,43 @@ define void @sext_i8_i16(ptr noalias nocapture readonly %p, ptr noalias nocaptur ; CHECK-COST: Cost of 1 for VF vscale x 2: WIDEN-CAST ir<%conv> = sext ir<%0> to i16 ; CHECK-COST: Cost of 1 for VF vscale x 4: WIDEN-CAST ir<%conv> = sext ir<%0> to i16 ; CHECK-COST: Cost of 0 for VF vscale x 8: WIDEN-CAST ir<%conv> = sext ir<%0> to i16 -; CHECK-LABEL: define void @sext_i8_i16 -; CHECK-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[LEN:%.*]]) #[[ATTR0]] { -; CHECK-NEXT: entry: +; CHECK-LABEL: define void @sext_i8_i16( +; CHECK-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i32 [[LEN:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*]]: ; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[LEN]], -1 ; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 ; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 ; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 4 ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]] -; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] -; CHECK: vector.ph: -; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP3]], 4 -; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]] +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP3]], 4 +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP5]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] -; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] -; CHECK: vector.body: -; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP7]], align 1 -; CHECK-NEXT: [[TMP8:%.*]] = sext [[WIDE_LOAD]] to -; CHECK-NEXT: [[TMP9:%.*]] = add [[TMP8]], splat (i16 2) -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i16, ptr [[Q]], i64 [[INDEX]] -; CHECK-NEXT: store [[TMP9]], ptr [[TMP10]], align 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]] -; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] -; CHECK: middle.block: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 1 +; CHECK-NEXT: [[TMP7:%.*]] = sext [[WIDE_LOAD]] to +; CHECK-NEXT: [[TMP8:%.*]] = add [[TMP7]], splat (i16 2) +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, ptr [[Q]], i64 [[INDEX]] +; CHECK-NEXT: store [[TMP8]], ptr [[TMP9]], align 2 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] +; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] -; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] -; CHECK: scalar.ph: -; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] -; CHECK-NEXT: br label [[FOR_BODY:%.*]] -; CHECK: for.body: -; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] +; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] +; CHECK-NEXT: br label %[[FOR_BODY:.*]] +; CHECK: [[FOR_BODY]]: +; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ] ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDVARS_IV]] -; CHECK-NEXT: [[TMP12:%.*]] = load i8, ptr [[ARRAYIDX]], align 1 -; CHECK-NEXT: [[CONV:%.*]] = sext i8 [[TMP12]] to i32 +; CHECK-NEXT: [[TMP11:%.*]] = load i8, ptr [[ARRAYIDX]], align 1 +; CHECK-NEXT: [[CONV:%.*]] = sext i8 [[TMP11]] to i32 ; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i32 [[CONV]], 2 ; CHECK-NEXT: [[CONV1:%.*]] = trunc i32 [[ADD]] to i16 ; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i16, ptr [[Q]], i64 [[INDVARS_IV]] @@ -140,8 +140,8 @@ define void @sext_i8_i16(ptr noalias nocapture readonly %p, ptr noalias nocaptur ; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 ; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 ; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[LEN]] -; CHECK-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] -; CHECK: exit: +; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] +; CHECK: [[EXIT]]: ; CHECK-NEXT: ret void ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll index da32ef49ddb14..1f66bca32d411 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll @@ -1,4 +1,4 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph:" --version 2 +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 6 ; RUN: opt -mtriple=riscv64-none-linux-gnu -S -passes=loop-vectorize -mattr=+v -tail-folding-policy=dont-fold-tail %s 2>&1 | FileCheck %s -check-prefix=SCALAR_EPILOGUE ; RUN: opt -mtriple=riscv64-none-linux-gnu -S -passes=loop-vectorize -mattr=+v -tail-folding-policy=prefer-fold-tail -force-tail-folding-style=data %s 2>&1 | FileCheck %s -check-prefix=PREDICATED_DATA ; RUN: opt -mtriple=riscv64-none-linux-gnu -S -passes=loop-vectorize -mattr=+v -tail-folding-policy=prefer-fold-tail %s 2>&1 | FileCheck %s -check-prefix=PREDICATED_DATA-WITH-EVL @@ -6,16 +6,16 @@ target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 zeroext %guard) { -; SCALAR_EPILOGUE-LABEL: define void @masked_strided_factor2 -; SCALAR_EPILOGUE-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0:[0-9]+]] { -; SCALAR_EPILOGUE-NEXT: entry: +; SCALAR_EPILOGUE-LABEL: define void @masked_strided_factor2( +; SCALAR_EPILOGUE-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0:[0-9]+]] { +; SCALAR_EPILOGUE-NEXT: [[ENTRY:.*:]] ; SCALAR_EPILOGUE-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32 ; SCALAR_EPILOGUE-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() ; SCALAR_EPILOGUE-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4 -; SCALAR_EPILOGUE-NEXT: [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP1]], i32 32) -; SCALAR_EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 1024, [[UMAX]] -; SCALAR_EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] -; SCALAR_EPILOGUE: vector.ph: +; SCALAR_EPILOGUE-NEXT: [[TMP2:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP1]], i32 32) +; SCALAR_EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 1024, [[TMP2]] +; SCALAR_EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; SCALAR_EPILOGUE: [[VECTOR_PH]]: ; SCALAR_EPILOGUE-NEXT: [[TMP3:%.*]] = shl nuw i32 [[TMP0]], 4 ; SCALAR_EPILOGUE-NEXT: [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]] ; SCALAR_EPILOGUE-NEXT: [[N_VEC:%.*]] = sub i32 1024, [[N_MOD_VF]] @@ -24,10 +24,10 @@ define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noali ; SCALAR_EPILOGUE-NEXT: [[TMP4:%.*]] = call @llvm.stepvector.nxv16i32() ; SCALAR_EPILOGUE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement poison, i32 [[TMP3]], i64 0 ; SCALAR_EPILOGUE-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector [[BROADCAST_SPLATINSERT1]], poison, zeroinitializer -; SCALAR_EPILOGUE-NEXT: br label [[VECTOR_BODY:%.*]] -; SCALAR_EPILOGUE: vector.body: -; SCALAR_EPILOGUE-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; SCALAR_EPILOGUE-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; SCALAR_EPILOGUE-NEXT: br label %[[VECTOR_BODY:.*]] +; SCALAR_EPILOGUE: [[VECTOR_BODY]]: +; SCALAR_EPILOGUE-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; SCALAR_EPILOGUE-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] ; SCALAR_EPILOGUE-NEXT: [[TMP5:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT]] ; SCALAR_EPILOGUE-NEXT: [[TMP6:%.*]] = shl i32 [[INDEX]], 1 ; SCALAR_EPILOGUE-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i32 [[TMP6]] @@ -46,18 +46,18 @@ define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noali ; SCALAR_EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]] ; SCALAR_EPILOGUE-NEXT: [[VEC_IND_NEXT]] = add nuw nsw [[VEC_IND]], [[BROADCAST_SPLAT2]] ; SCALAR_EPILOGUE-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; SCALAR_EPILOGUE-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] -; SCALAR_EPILOGUE: middle.block: +; SCALAR_EPILOGUE-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; SCALAR_EPILOGUE: [[MIDDLE_BLOCK]]: ; SCALAR_EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i32 1024, [[N_VEC]] -; SCALAR_EPILOGUE-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] -; SCALAR_EPILOGUE: scalar.ph: +; SCALAR_EPILOGUE-NEXT: br i1 [[CMP_N]], [[FOR_END:label %.*]], label %[[SCALAR_PH]] +; SCALAR_EPILOGUE: [[SCALAR_PH]]: ; -; PREDICATED_DATA-LABEL: define void @masked_strided_factor2 -; PREDICATED_DATA-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0:[0-9]+]] { -; PREDICATED_DATA-NEXT: entry: +; PREDICATED_DATA-LABEL: define void @masked_strided_factor2( +; PREDICATED_DATA-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0:[0-9]+]] { +; PREDICATED_DATA-NEXT: [[ENTRY:.*:]] ; PREDICATED_DATA-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32 -; PREDICATED_DATA-NEXT: br label [[VECTOR_PH:%.*]] -; PREDICATED_DATA: vector.ph: +; PREDICATED_DATA-NEXT: br label %[[VECTOR_PH:.*]] +; PREDICATED_DATA: [[VECTOR_PH]]: ; PREDICATED_DATA-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() ; PREDICATED_DATA-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4 ; PREDICATED_DATA-NEXT: [[TMP2:%.*]] = sub i32 [[TMP1]], 1 @@ -69,10 +69,10 @@ define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noali ; PREDICATED_DATA-NEXT: [[TMP3:%.*]] = call @llvm.stepvector.nxv16i32() ; PREDICATED_DATA-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement poison, i32 [[TMP1]], i64 0 ; PREDICATED_DATA-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector [[BROADCAST_SPLATINSERT1]], poison, zeroinitializer -; PREDICATED_DATA-NEXT: br label [[VECTOR_BODY:%.*]] -; PREDICATED_DATA: vector.body: -; PREDICATED_DATA-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; PREDICATED_DATA-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP3]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; PREDICATED_DATA-NEXT: br label %[[VECTOR_BODY:.*]] +; PREDICATED_DATA: [[VECTOR_BODY]]: +; PREDICATED_DATA-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; PREDICATED_DATA-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP3]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] ; PREDICATED_DATA-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 1024) ; PREDICATED_DATA-NEXT: [[TMP4:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT]] ; PREDICATED_DATA-NEXT: [[TMP5:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP4]], zeroinitializer @@ -93,26 +93,26 @@ define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noali ; PREDICATED_DATA-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]] ; PREDICATED_DATA-NEXT: [[VEC_IND_NEXT]] = add nuw nsw [[VEC_IND]], [[BROADCAST_SPLAT2]] ; PREDICATED_DATA-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; PREDICATED_DATA-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] -; PREDICATED_DATA: middle.block: -; PREDICATED_DATA-NEXT: br label [[FOR_END:%.*]] -; PREDICATED_DATA: for.end: +; PREDICATED_DATA-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; PREDICATED_DATA: [[MIDDLE_BLOCK]]: +; PREDICATED_DATA-NEXT: br label %[[FOR_END:.*]] +; PREDICATED_DATA: [[FOR_END]]: ; PREDICATED_DATA-NEXT: ret void ; -; PREDICATED_DATA-WITH-EVL-LABEL: define void @masked_strided_factor2 -; PREDICATED_DATA-WITH-EVL-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0:[0-9]+]] { -; PREDICATED_DATA-WITH-EVL-NEXT: entry: +; PREDICATED_DATA-WITH-EVL-LABEL: define void @masked_strided_factor2( +; PREDICATED_DATA-WITH-EVL-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0:[0-9]+]] { +; PREDICATED_DATA-WITH-EVL-NEXT: [[ENTRY:.*:]] ; PREDICATED_DATA-WITH-EVL-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32 -; PREDICATED_DATA-WITH-EVL-NEXT: br label [[VECTOR_PH:%.*]] -; PREDICATED_DATA-WITH-EVL: vector.ph: +; PREDICATED_DATA-WITH-EVL-NEXT: br label %[[VECTOR_PH:.*]] +; PREDICATED_DATA-WITH-EVL: [[VECTOR_PH]]: ; PREDICATED_DATA-WITH-EVL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[CONV]], i64 0 ; PREDICATED_DATA-WITH-EVL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; PREDICATED_DATA-WITH-EVL-NEXT: [[TMP0:%.*]] = call @llvm.stepvector.nxv16i32() -; PREDICATED_DATA-WITH-EVL-NEXT: br label [[VECTOR_BODY:%.*]] -; PREDICATED_DATA-WITH-EVL: vector.body: -; PREDICATED_DATA-WITH-EVL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ] -; PREDICATED_DATA-WITH-EVL-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP0]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] -; PREDICATED_DATA-WITH-EVL-NEXT: [[AVL:%.*]] = phi i32 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ] +; PREDICATED_DATA-WITH-EVL-NEXT: br label %[[VECTOR_BODY:.*]] +; PREDICATED_DATA-WITH-EVL: [[VECTOR_BODY]]: +; PREDICATED_DATA-WITH-EVL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ] +; PREDICATED_DATA-WITH-EVL-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP0]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] +; PREDICATED_DATA-WITH-EVL-NEXT: [[AVL:%.*]] = phi i32 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ] ; PREDICATED_DATA-WITH-EVL-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 16, i1 true) ; PREDICATED_DATA-WITH-EVL-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement poison, i32 [[TMP1]], i64 0 ; PREDICATED_DATA-WITH-EVL-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector [[BROADCAST_SPLATINSERT1]], poison, zeroinitializer @@ -137,10 +137,10 @@ define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noali ; PREDICATED_DATA-WITH-EVL-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP1]] ; PREDICATED_DATA-WITH-EVL-NEXT: [[VEC_IND_NEXT]] = add nuw nsw [[VEC_IND]], [[BROADCAST_SPLAT2]] ; PREDICATED_DATA-WITH-EVL-NEXT: [[TMP11:%.*]] = icmp eq i32 [[AVL_NEXT]], 0 -; PREDICATED_DATA-WITH-EVL-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] -; PREDICATED_DATA-WITH-EVL: middle.block: -; PREDICATED_DATA-WITH-EVL-NEXT: br label [[FOR_END:%.*]] -; PREDICATED_DATA-WITH-EVL: for.end: +; PREDICATED_DATA-WITH-EVL-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; PREDICATED_DATA-WITH-EVL: [[MIDDLE_BLOCK]]: +; PREDICATED_DATA-WITH-EVL-NEXT: br label %[[FOR_END:.*]] +; PREDICATED_DATA-WITH-EVL: [[FOR_END]]: ; PREDICATED_DATA-WITH-EVL-NEXT: ret void ; entry: @@ -179,16 +179,16 @@ for.end: define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noalias nocapture %q, i8 zeroext %guard) { -; SCALAR_EPILOGUE-LABEL: define void @masked_strided_factor4 -; SCALAR_EPILOGUE-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] { -; SCALAR_EPILOGUE-NEXT: entry: +; SCALAR_EPILOGUE-LABEL: define void @masked_strided_factor4( +; SCALAR_EPILOGUE-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] { +; SCALAR_EPILOGUE-NEXT: [[ENTRY:.*:]] ; SCALAR_EPILOGUE-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32 ; SCALAR_EPILOGUE-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() ; SCALAR_EPILOGUE-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4 -; SCALAR_EPILOGUE-NEXT: [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP1]], i32 32) -; SCALAR_EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 1024, [[UMAX]] -; SCALAR_EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] -; SCALAR_EPILOGUE: vector.ph: +; SCALAR_EPILOGUE-NEXT: [[TMP2:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP1]], i32 32) +; SCALAR_EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 1024, [[TMP2]] +; SCALAR_EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; SCALAR_EPILOGUE: [[VECTOR_PH]]: ; SCALAR_EPILOGUE-NEXT: [[TMP3:%.*]] = shl nuw i32 [[TMP0]], 4 ; SCALAR_EPILOGUE-NEXT: [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]] ; SCALAR_EPILOGUE-NEXT: [[N_VEC:%.*]] = sub i32 1024, [[N_MOD_VF]] @@ -197,10 +197,10 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali ; SCALAR_EPILOGUE-NEXT: [[TMP4:%.*]] = call @llvm.stepvector.nxv16i32() ; SCALAR_EPILOGUE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement poison, i32 [[TMP3]], i64 0 ; SCALAR_EPILOGUE-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector [[BROADCAST_SPLATINSERT1]], poison, zeroinitializer -; SCALAR_EPILOGUE-NEXT: br label [[VECTOR_BODY:%.*]] -; SCALAR_EPILOGUE: vector.body: -; SCALAR_EPILOGUE-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; SCALAR_EPILOGUE-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; SCALAR_EPILOGUE-NEXT: br label %[[VECTOR_BODY:.*]] +; SCALAR_EPILOGUE: [[VECTOR_BODY]]: +; SCALAR_EPILOGUE-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; SCALAR_EPILOGUE-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] ; SCALAR_EPILOGUE-NEXT: [[TMP5:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT]] ; SCALAR_EPILOGUE-NEXT: [[TMP6:%.*]] = shl i32 [[INDEX]], 2 ; SCALAR_EPILOGUE-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i32 [[TMP6]] @@ -224,18 +224,18 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali ; SCALAR_EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]] ; SCALAR_EPILOGUE-NEXT: [[VEC_IND_NEXT]] = add nuw nsw [[VEC_IND]], [[BROADCAST_SPLAT2]] ; SCALAR_EPILOGUE-NEXT: [[TMP19:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; SCALAR_EPILOGUE-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] -; SCALAR_EPILOGUE: middle.block: +; SCALAR_EPILOGUE-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; SCALAR_EPILOGUE: [[MIDDLE_BLOCK]]: ; SCALAR_EPILOGUE-NEXT: [[CMP_N:%.*]] = icmp eq i32 1024, [[N_VEC]] -; SCALAR_EPILOGUE-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] -; SCALAR_EPILOGUE: scalar.ph: +; SCALAR_EPILOGUE-NEXT: br i1 [[CMP_N]], [[FOR_END:label %.*]], label %[[SCALAR_PH]] +; SCALAR_EPILOGUE: [[SCALAR_PH]]: ; -; PREDICATED_DATA-LABEL: define void @masked_strided_factor4 -; PREDICATED_DATA-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] { -; PREDICATED_DATA-NEXT: entry: +; PREDICATED_DATA-LABEL: define void @masked_strided_factor4( +; PREDICATED_DATA-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] { +; PREDICATED_DATA-NEXT: [[ENTRY:.*:]] ; PREDICATED_DATA-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32 -; PREDICATED_DATA-NEXT: br label [[VECTOR_PH:%.*]] -; PREDICATED_DATA: vector.ph: +; PREDICATED_DATA-NEXT: br label %[[VECTOR_PH:.*]] +; PREDICATED_DATA: [[VECTOR_PH]]: ; PREDICATED_DATA-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() ; PREDICATED_DATA-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4 ; PREDICATED_DATA-NEXT: [[TMP2:%.*]] = sub i32 [[TMP1]], 1 @@ -247,10 +247,10 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali ; PREDICATED_DATA-NEXT: [[TMP3:%.*]] = call @llvm.stepvector.nxv16i32() ; PREDICATED_DATA-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement poison, i32 [[TMP1]], i64 0 ; PREDICATED_DATA-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector [[BROADCAST_SPLATINSERT1]], poison, zeroinitializer -; PREDICATED_DATA-NEXT: br label [[VECTOR_BODY:%.*]] -; PREDICATED_DATA: vector.body: -; PREDICATED_DATA-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; PREDICATED_DATA-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP3]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; PREDICATED_DATA-NEXT: br label %[[VECTOR_BODY:.*]] +; PREDICATED_DATA: [[VECTOR_BODY]]: +; PREDICATED_DATA-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; PREDICATED_DATA-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP3]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] ; PREDICATED_DATA-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 1024) ; PREDICATED_DATA-NEXT: [[TMP4:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT]] ; PREDICATED_DATA-NEXT: [[TMP5:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP4]], zeroinitializer @@ -276,26 +276,26 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali ; PREDICATED_DATA-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]] ; PREDICATED_DATA-NEXT: [[VEC_IND_NEXT]] = add nuw nsw [[VEC_IND]], [[BROADCAST_SPLAT2]] ; PREDICATED_DATA-NEXT: [[TMP19:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; PREDICATED_DATA-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] -; PREDICATED_DATA: middle.block: -; PREDICATED_DATA-NEXT: br label [[FOR_END:%.*]] -; PREDICATED_DATA: for.end: +; PREDICATED_DATA-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] +; PREDICATED_DATA: [[MIDDLE_BLOCK]]: +; PREDICATED_DATA-NEXT: br label %[[FOR_END:.*]] +; PREDICATED_DATA: [[FOR_END]]: ; PREDICATED_DATA-NEXT: ret void ; -; PREDICATED_DATA-WITH-EVL-LABEL: define void @masked_strided_factor4 -; PREDICATED_DATA-WITH-EVL-SAME: (ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] { -; PREDICATED_DATA-WITH-EVL-NEXT: entry: +; PREDICATED_DATA-WITH-EVL-LABEL: define void @masked_strided_factor4( +; PREDICATED_DATA-WITH-EVL-SAME: ptr noalias readonly captures(none) [[P:%.*]], ptr noalias captures(none) [[Q:%.*]], i8 zeroext [[GUARD:%.*]]) #[[ATTR0]] { +; PREDICATED_DATA-WITH-EVL-NEXT: [[ENTRY:.*:]] ; PREDICATED_DATA-WITH-EVL-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32 -; PREDICATED_DATA-WITH-EVL-NEXT: br label [[VECTOR_PH:%.*]] -; PREDICATED_DATA-WITH-EVL: vector.ph: +; PREDICATED_DATA-WITH-EVL-NEXT: br label %[[VECTOR_PH:.*]] +; PREDICATED_DATA-WITH-EVL: [[VECTOR_PH]]: ; PREDICATED_DATA-WITH-EVL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[CONV]], i64 0 ; PREDICATED_DATA-WITH-EVL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; PREDICATED_DATA-WITH-EVL-NEXT: [[TMP0:%.*]] = call @llvm.stepvector.nxv16i32() -; PREDICATED_DATA-WITH-EVL-NEXT: br label [[VECTOR_BODY:%.*]] -; PREDICATED_DATA-WITH-EVL: vector.body: -; PREDICATED_DATA-WITH-EVL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ] -; PREDICATED_DATA-WITH-EVL-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP0]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] -; PREDICATED_DATA-WITH-EVL-NEXT: [[AVL:%.*]] = phi i32 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ] +; PREDICATED_DATA-WITH-EVL-NEXT: br label %[[VECTOR_BODY:.*]] +; PREDICATED_DATA-WITH-EVL: [[VECTOR_BODY]]: +; PREDICATED_DATA-WITH-EVL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ] +; PREDICATED_DATA-WITH-EVL-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP0]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] +; PREDICATED_DATA-WITH-EVL-NEXT: [[AVL:%.*]] = phi i32 [ 1024, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ] ; PREDICATED_DATA-WITH-EVL-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 16, i1 true) ; PREDICATED_DATA-WITH-EVL-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement poison, i32 [[TMP1]], i64 0 ; PREDICATED_DATA-WITH-EVL-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector [[BROADCAST_SPLATINSERT1]], poison, zeroinitializer @@ -325,10 +325,10 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali ; PREDICATED_DATA-WITH-EVL-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP1]] ; PREDICATED_DATA-WITH-EVL-NEXT: [[VEC_IND_NEXT]] = add nuw nsw [[VEC_IND]], [[BROADCAST_SPLAT2]] ; PREDICATED_DATA-WITH-EVL-NEXT: [[TMP16:%.*]] = icmp eq i32 [[AVL_NEXT]], 0 -; PREDICATED_DATA-WITH-EVL-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] -; PREDICATED_DATA-WITH-EVL: middle.block: -; PREDICATED_DATA-WITH-EVL-NEXT: br label [[FOR_END:%.*]] -; PREDICATED_DATA-WITH-EVL: for.end: +; PREDICATED_DATA-WITH-EVL-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] +; PREDICATED_DATA-WITH-EVL: [[MIDDLE_BLOCK]]: +; PREDICATED_DATA-WITH-EVL-NEXT: br label %[[FOR_END:.*]] +; PREDICATED_DATA-WITH-EVL: [[FOR_END]]: ; PREDICATED_DATA-WITH-EVL-NEXT: ret void ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll index 185dc65be98ed..fd0e130ee3980 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll @@ -14,25 +14,25 @@ define void @interleave(ptr noalias %a, ptr noalias %b, i64 %N) { ; IF-EVL: vector.ph: ; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]] ; IF-EVL: vector.body: -; IF-EVL-NEXT: [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ] ; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ] -; IF-EVL-NEXT: [[TMP16:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) -; IF-EVL-NEXT: [[TMP6:%.*]] = getelementptr inbounds [2 x i32], ptr [[B:%.*]], i64 [[EVL_BASED_IV]], i32 0 -; IF-EVL-NEXT: [[INTERLEAVE_EVL:%.*]] = mul nuw nsw i32 [[TMP16]], 2 -; IF-EVL-NEXT: [[WIDE_VEC:%.*]] = call @llvm.vp.load.nxv8i32.p0(ptr align 4 [[TMP6]], splat (i1 true), i32 [[INTERLEAVE_EVL]]) -; IF-EVL-NEXT: [[STRIDED_VEC:%.*]] = call { , } @llvm.vector.deinterleave2.nxv8i32( [[WIDE_VEC]]) -; IF-EVL-NEXT: [[TMP14:%.*]] = extractvalue { , } [[STRIDED_VEC]], 0 -; IF-EVL-NEXT: [[TMP15:%.*]] = extractvalue { , } [[STRIDED_VEC]], 1 -; IF-EVL-NEXT: [[TMP9:%.*]] = add nsw [[TMP15]], [[TMP14]] -; IF-EVL-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]] -; IF-EVL-NEXT: call void @llvm.vp.store.nxv4i32.p0( [[TMP9]], ptr align 4 [[TMP10]], splat (i1 true), i32 [[TMP16]]) -; IF-EVL-NEXT: [[TMP8:%.*]] = zext i32 [[TMP16]] to i64 -; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP8]], [[EVL_BASED_IV]] -; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]] +; IF-EVL-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) +; IF-EVL-NEXT: [[TMP1:%.*]] = getelementptr inbounds [2 x i32], ptr [[B:%.*]], i64 [[INDEX]], i32 0 +; IF-EVL-NEXT: [[INTERLEAVE_EVL:%.*]] = mul nuw nsw i32 [[TMP0]], 2 +; IF-EVL-NEXT: [[WIDE_VP_LOAD:%.*]] = call @llvm.vp.load.nxv8i32.p0(ptr align 4 [[TMP1]], splat (i1 true), i32 [[INTERLEAVE_EVL]]) +; IF-EVL-NEXT: [[STRIDED_VEC:%.*]] = call { , } @llvm.vector.deinterleave2.nxv8i32( [[WIDE_VP_LOAD]]) +; IF-EVL-NEXT: [[TMP2:%.*]] = extractvalue { , } [[STRIDED_VEC]], 0 +; IF-EVL-NEXT: [[TMP3:%.*]] = extractvalue { , } [[STRIDED_VEC]], 1 +; IF-EVL-NEXT: [[TMP4:%.*]] = add nsw [[TMP3]], [[TMP2]] +; IF-EVL-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]] +; IF-EVL-NEXT: call void @llvm.vp.store.nxv4i32.p0( [[TMP4]], ptr align 4 [[TMP5]], splat (i1 true), i32 [[TMP0]]) +; IF-EVL-NEXT: [[TMP6:%.*]] = zext i32 [[TMP0]] to i64 +; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP6]], [[INDEX]] +; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP6]] ; IF-EVL-NEXT: [[TMP7:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 ; IF-EVL-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; IF-EVL: middle.block: -; IF-EVL-NEXT: br label [[FOR_BODY:%.*]] +; IF-EVL-NEXT: br label [[FOR_COND_CLEANUP:%.*]] ; IF-EVL: for.cond.cleanup: ; IF-EVL-NEXT: ret void ; @@ -43,23 +43,23 @@ define void @interleave(ptr noalias %a, ptr noalias %b, i64 %N) { ; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]] ; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; NO-VP: vector.ph: -; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2 -; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] +; NO-VP-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2 +; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]] ; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] ; NO-VP-NEXT: br label [[VECTOR_BODY:%.*]] ; NO-VP: vector.body: ; NO-VP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; NO-VP-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x i32], ptr [[B:%.*]], i64 [[INDEX]], i32 0 -; NO-VP-NEXT: [[WIDE_VEC1:%.*]] = load , ptr [[TMP13]], align 4 -; NO-VP-NEXT: [[STRIDED_VEC2:%.*]] = call { , } @llvm.vector.deinterleave2.nxv8i32( [[WIDE_VEC1]]) -; NO-VP-NEXT: [[TMP18:%.*]] = extractvalue { , } [[STRIDED_VEC2]], 0 -; NO-VP-NEXT: [[TMP19:%.*]] = extractvalue { , } [[STRIDED_VEC2]], 1 -; NO-VP-NEXT: [[TMP21:%.*]] = add nsw [[TMP19]], [[TMP18]] -; NO-VP-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]] -; NO-VP-NEXT: store [[TMP21]], ptr [[TMP22]], align 4 -; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]] -; NO-VP-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; NO-VP-NEXT: br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; NO-VP-NEXT: [[TMP3:%.*]] = getelementptr inbounds [2 x i32], ptr [[B:%.*]], i64 [[INDEX]], i32 0 +; NO-VP-NEXT: [[WIDE_VEC:%.*]] = load , ptr [[TMP3]], align 4 +; NO-VP-NEXT: [[STRIDED_VEC:%.*]] = call { , } @llvm.vector.deinterleave2.nxv8i32( [[WIDE_VEC]]) +; NO-VP-NEXT: [[TMP4:%.*]] = extractvalue { , } [[STRIDED_VEC]], 0 +; NO-VP-NEXT: [[TMP5:%.*]] = extractvalue { , } [[STRIDED_VEC]], 1 +; NO-VP-NEXT: [[TMP6:%.*]] = add nsw [[TMP5]], [[TMP4]] +; NO-VP-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]] +; NO-VP-NEXT: store [[TMP6]], ptr [[TMP7]], align 4 +; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]] +; NO-VP-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; NO-VP-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; NO-VP: middle.block: ; NO-VP-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] ; NO-VP-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]] @@ -69,10 +69,10 @@ define void @interleave(ptr noalias %a, ptr noalias %b, i64 %N) { ; NO-VP: for.body: ; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] ; NO-VP-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[B]], i64 [[IV]], i32 0 -; NO-VP-NEXT: [[TMP29:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +; NO-VP-NEXT: [[TMP9:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 ; NO-VP-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [2 x i32], ptr [[B]], i64 [[IV]], i32 1 -; NO-VP-NEXT: [[TMP30:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4 -; NO-VP-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP30]], [[TMP29]] +; NO-VP-NEXT: [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4 +; NO-VP-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP10]], [[TMP9]] ; NO-VP-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]] ; NO-VP-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX4]], align 4 ; NO-VP-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 @@ -119,38 +119,38 @@ define i32 @load_factor_4_with_gap(i64 %n, ptr noalias %a) { ; IF-EVL: vector.ph: ; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]] ; IF-EVL: vector.body: -; IF-EVL-NEXT: [[CURRENT_ITERATION_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ] -; IF-EVL-NEXT: [[VEC_PHI:%.*]] = phi [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[VEC_PHI:%.*]] = phi [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[VECTOR_BODY]] ] ; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ] -; IF-EVL-NEXT: [[TMP4:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) -; IF-EVL-NEXT: [[TMP1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], i64 [[CURRENT_ITERATION_IV]], i32 0 -; IF-EVL-NEXT: [[INTERLEAVE_EVL:%.*]] = mul nuw nsw i32 [[TMP4]], 4 +; IF-EVL-NEXT: [[TMP0:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) +; IF-EVL-NEXT: [[TMP1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], i64 [[INDEX]], i32 0 +; IF-EVL-NEXT: [[INTERLEAVE_EVL:%.*]] = mul nuw nsw i32 [[TMP0]], 4 ; IF-EVL-NEXT: [[WIDE_VP_LOAD:%.*]] = call @llvm.vp.load.nxv16i32.p0(ptr align 4 [[TMP1]], splat (i1 true), i32 [[INTERLEAVE_EVL]]) ; IF-EVL-NEXT: [[STRIDED_VEC:%.*]] = call { , , , } @llvm.vector.deinterleave4.nxv16i32( [[WIDE_VP_LOAD]]) -; IF-EVL-NEXT: [[WIDE_MASKED_GATHER:%.*]] = extractvalue { , , , } [[STRIDED_VEC]], 0 -; IF-EVL-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = extractvalue { , , , } [[STRIDED_VEC]], 1 -; IF-EVL-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = extractvalue { , , , } [[STRIDED_VEC]], 3 -; IF-EVL-NEXT: [[TMP8:%.*]] = add [[VEC_PHI]], [[WIDE_MASKED_GATHER]] -; IF-EVL-NEXT: [[TMP9:%.*]] = add [[TMP8]], [[WIDE_MASKED_GATHER1]] -; IF-EVL-NEXT: [[TMP11:%.*]] = add [[TMP9]], [[WIDE_MASKED_GATHER2]] -; IF-EVL-NEXT: [[TMP12]] = call @llvm.vp.merge.nxv4i32( splat (i1 true), [[TMP11]], [[VEC_PHI]], i32 [[TMP4]]) -; IF-EVL-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64 -; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP5]], [[CURRENT_ITERATION_IV]] -; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]] -; IF-EVL-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 -; IF-EVL-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] +; IF-EVL-NEXT: [[TMP2:%.*]] = extractvalue { , , , } [[STRIDED_VEC]], 0 +; IF-EVL-NEXT: [[TMP3:%.*]] = extractvalue { , , , } [[STRIDED_VEC]], 1 +; IF-EVL-NEXT: [[TMP4:%.*]] = extractvalue { , , , } [[STRIDED_VEC]], 3 +; IF-EVL-NEXT: [[TMP5:%.*]] = add [[VEC_PHI]], [[TMP2]] +; IF-EVL-NEXT: [[TMP6:%.*]] = add [[TMP5]], [[TMP3]] +; IF-EVL-NEXT: [[TMP7:%.*]] = add [[TMP6]], [[TMP4]] +; IF-EVL-NEXT: [[TMP8]] = call @llvm.vp.merge.nxv4i32( splat (i1 true), [[TMP7]], [[VEC_PHI]], i32 [[TMP0]]) +; IF-EVL-NEXT: [[TMP9:%.*]] = zext i32 [[TMP0]] to i64 +; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP9]], [[INDEX]] +; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]] +; IF-EVL-NEXT: [[TMP10:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 +; IF-EVL-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] ; IF-EVL: middle.block: -; IF-EVL-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32( [[TMP12]]) -; IF-EVL-NEXT: br label [[FOR_BODY:%.*]] +; IF-EVL-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32( [[TMP8]]) +; IF-EVL-NEXT: br label [[EXIT:%.*]] ; IF-EVL: exit: -; IF-EVL-NEXT: ret i32 [[TMP15]] +; IF-EVL-NEXT: ret i32 [[TMP11]] ; ; NO-VP-LABEL: @load_factor_4_with_gap( ; NO-VP-NEXT: entry: ; NO-VP-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() ; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2 -; NO-VP-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 8) -; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[UMAX]] +; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 8) +; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP2]] ; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; NO-VP: vector.ph: ; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2 @@ -159,44 +159,44 @@ define i32 @load_factor_4_with_gap(i64 %n, ptr noalias %a) { ; NO-VP-NEXT: br label [[VECTOR_BODY:%.*]] ; NO-VP: vector.body: ; NO-VP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; NO-VP-NEXT: [[VEC_PHI:%.*]] = phi [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ] +; NO-VP-NEXT: [[VEC_PHI:%.*]] = phi [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ] ; NO-VP-NEXT: [[TMP4:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], i64 [[INDEX]], i32 0 ; NO-VP-NEXT: [[WIDE_VEC:%.*]] = load , ptr [[TMP4]], align 4 ; NO-VP-NEXT: [[STRIDED_VEC:%.*]] = call { , , , } @llvm.vector.deinterleave4.nxv16i32( [[WIDE_VEC]]) -; NO-VP-NEXT: [[WIDE_MASKED_GATHER:%.*]] = extractvalue { , , , } [[STRIDED_VEC]], 0 -; NO-VP-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = extractvalue { , , , } [[STRIDED_VEC]], 1 -; NO-VP-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = extractvalue { , , , } [[STRIDED_VEC]], 3 -; NO-VP-NEXT: [[TMP10:%.*]] = add [[VEC_PHI]], [[WIDE_MASKED_GATHER]] -; NO-VP-NEXT: [[TMP11:%.*]] = add [[TMP10]], [[WIDE_MASKED_GATHER1]] -; NO-VP-NEXT: [[TMP13]] = add [[TMP11]], [[WIDE_MASKED_GATHER2]] +; NO-VP-NEXT: [[TMP5:%.*]] = extractvalue { , , , } [[STRIDED_VEC]], 0 +; NO-VP-NEXT: [[TMP6:%.*]] = extractvalue { , , , } [[STRIDED_VEC]], 1 +; NO-VP-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[STRIDED_VEC]], 3 +; NO-VP-NEXT: [[TMP8:%.*]] = add [[VEC_PHI]], [[TMP5]] +; NO-VP-NEXT: [[TMP9:%.*]] = add [[TMP8]], [[TMP6]] +; NO-VP-NEXT: [[TMP10]] = add [[TMP9]], [[TMP7]] ; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]] -; NO-VP-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; NO-VP-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; NO-VP-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; NO-VP-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; NO-VP: middle.block: -; NO-VP-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32( [[TMP13]]) +; NO-VP-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32( [[TMP10]]) ; NO-VP-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] ; NO-VP-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] ; NO-VP: scalar.ph: ; NO-VP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] -; NO-VP-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP15]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] +; NO-VP-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP12]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] ; NO-VP-NEXT: br label [[FOR_BODY:%.*]] ; NO-VP: for.body: ; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] ; NO-VP-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD2:%.*]], [[FOR_BODY]] ] ; NO-VP-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 0 -; NO-VP-NEXT: [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 -; NO-VP-NEXT: [[ADD:%.*]] = add nsw i32 [[RDX]], [[TMP16]] +; NO-VP-NEXT: [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +; NO-VP-NEXT: [[ADD:%.*]] = add nsw i32 [[RDX]], [[TMP13]] ; NO-VP-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 1 -; NO-VP-NEXT: [[TMP17:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4 -; NO-VP-NEXT: [[ADD1:%.*]] = add nsw i32 [[ADD]], [[TMP17]] +; NO-VP-NEXT: [[TMP14:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4 +; NO-VP-NEXT: [[ADD1:%.*]] = add nsw i32 [[ADD]], [[TMP14]] ; NO-VP-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 3 -; NO-VP-NEXT: [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4 -; NO-VP-NEXT: [[ADD2]] = add nsw i32 [[ADD1]], [[TMP18]] +; NO-VP-NEXT: [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4 +; NO-VP-NEXT: [[ADD2]] = add nsw i32 [[ADD1]], [[TMP15]] ; NO-VP-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 ; NO-VP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] ; NO-VP-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; NO-VP: exit: -; NO-VP-NEXT: [[ADD2_LCSSA:%.*]] = phi i32 [ [[ADD2]], [[FOR_BODY]] ], [ [[TMP15]], [[MIDDLE_BLOCK]] ] +; NO-VP-NEXT: [[ADD2_LCSSA:%.*]] = phi i32 [ [[ADD2]], [[FOR_BODY]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ] ; NO-VP-NEXT: ret i32 [[ADD2_LCSSA]] ; entry: @@ -239,30 +239,30 @@ define void @store_factor_4_with_gap(i32 %n, ptr noalias %a) { ; IF-EVL: vector.ph: ; IF-EVL-NEXT: [[TMP0:%.*]] = getelementptr nuw i8, ptr [[A:%.*]], i64 4 ; IF-EVL-NEXT: [[TMP1:%.*]] = getelementptr nuw i8, ptr [[A]], i64 12 -; IF-EVL-NEXT: [[TMP4:%.*]] = call @llvm.stepvector.nxv4i32() +; IF-EVL-NEXT: [[TMP2:%.*]] = call @llvm.stepvector.nxv4i32() ; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]] ; IF-EVL: vector.body: ; IF-EVL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ] -; IF-EVL-NEXT: [[VEC_IND2:%.*]] = phi [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT5:%.*]], [[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP2]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] ; IF-EVL-NEXT: [[AVL:%.*]] = phi i32 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ] -; IF-EVL-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true) -; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[TMP6]], i64 0 +; IF-EVL-NEXT: [[TMP3:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true) +; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[TMP3]], i64 0 ; IF-EVL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer -; IF-EVL-NEXT: [[TMP9:%.*]] = zext i32 [[INDEX]] to i64 -; IF-EVL-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP9]], 4 -; IF-EVL-NEXT: [[TMP10:%.*]] = getelementptr nuw i8, ptr [[A]], i64 [[TMP5]] -; IF-EVL-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64( [[VEC_IND2]], ptr align 4 [[TMP10]], i64 16, splat (i1 true), i32 [[TMP6]]) +; IF-EVL-NEXT: [[TMP4:%.*]] = zext i32 [[INDEX]] to i64 +; IF-EVL-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 4 +; IF-EVL-NEXT: [[TMP6:%.*]] = getelementptr nuw i8, ptr [[A]], i64 [[TMP5]] +; IF-EVL-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64( [[VEC_IND]], ptr align 4 [[TMP6]], i64 16, splat (i1 true), i32 [[TMP3]]) ; IF-EVL-NEXT: [[TMP7:%.*]] = getelementptr nuw i8, ptr [[TMP0]], i64 [[TMP5]] -; IF-EVL-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64( [[VEC_IND2]], ptr align 4 [[TMP7]], i64 16, splat (i1 true), i32 [[TMP6]]) -; IF-EVL-NEXT: [[TMP11:%.*]] = getelementptr nuw i8, ptr [[TMP1]], i64 [[TMP5]] -; IF-EVL-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64( [[VEC_IND2]], ptr align 4 [[TMP11]], i64 16, splat (i1 true), i32 [[TMP6]]) -; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add i32 [[TMP6]], [[INDEX]] -; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP6]] -; IF-EVL-NEXT: [[VEC_IND_NEXT5]] = add nuw nsw [[VEC_IND2]], [[BROADCAST_SPLAT]] -; IF-EVL-NEXT: [[TMP8:%.*]] = icmp eq i32 [[AVL_NEXT]], 0 -; IF-EVL-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; IF-EVL-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64( [[VEC_IND]], ptr align 4 [[TMP7]], i64 16, splat (i1 true), i32 [[TMP3]]) +; IF-EVL-NEXT: [[TMP8:%.*]] = getelementptr nuw i8, ptr [[TMP1]], i64 [[TMP5]] +; IF-EVL-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64( [[VEC_IND]], ptr align 4 [[TMP8]], i64 16, splat (i1 true), i32 [[TMP3]]) +; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add i32 [[TMP3]], [[INDEX]] +; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP3]] +; IF-EVL-NEXT: [[VEC_IND_NEXT]] = add nuw nsw [[VEC_IND]], [[BROADCAST_SPLAT]] +; IF-EVL-NEXT: [[TMP9:%.*]] = icmp eq i32 [[AVL_NEXT]], 0 +; IF-EVL-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; IF-EVL: middle.block: -; IF-EVL-NEXT: br label [[FOR_BODY:%.*]] +; IF-EVL-NEXT: br label [[EXIT:%.*]] ; IF-EVL: exit: ; IF-EVL-NEXT: ret void ; @@ -270,34 +270,34 @@ define void @store_factor_4_with_gap(i32 %n, ptr noalias %a) { ; NO-VP-NEXT: entry: ; NO-VP-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() ; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2 -; NO-VP-NEXT: [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP1]], i32 8) -; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], [[UMAX]] +; NO-VP-NEXT: [[TMP2:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP1]], i32 8) +; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], [[TMP2]] ; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; NO-VP: vector.ph: -; NO-VP-NEXT: [[TMP9:%.*]] = shl nuw i32 [[TMP0]], 2 -; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], [[TMP9]] +; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i32 [[TMP0]], 2 +; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], [[TMP3]] ; NO-VP-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]] ; NO-VP-NEXT: [[TMP4:%.*]] = getelementptr nuw i8, ptr [[A:%.*]], i64 4 ; NO-VP-NEXT: [[TMP5:%.*]] = getelementptr nuw i8, ptr [[A]], i64 12 -; NO-VP-NEXT: [[TMP7:%.*]] = call @llvm.stepvector.nxv4i32() -; NO-VP-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement poison, i32 [[TMP9]], i64 0 -; NO-VP-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector [[BROADCAST_SPLATINSERT2]], poison, zeroinitializer +; NO-VP-NEXT: [[TMP6:%.*]] = call @llvm.stepvector.nxv4i32() +; NO-VP-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[TMP3]], i64 0 +; NO-VP-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; NO-VP-NEXT: br label [[VECTOR_BODY:%.*]] ; NO-VP: vector.body: ; NO-VP-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; NO-VP-NEXT: [[VEC_IND4:%.*]] = phi [ [[TMP7]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT5:%.*]], [[VECTOR_BODY]] ] -; NO-VP-NEXT: [[TMP12:%.*]] = zext i32 [[INDEX]] to i64 -; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP12]], 4 -; NO-VP-NEXT: [[TMP13:%.*]] = getelementptr nuw i8, ptr [[A]], i64 [[TMP8]] -; NO-VP-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64( [[VEC_IND4]], ptr align 4 [[TMP13]], i64 16, splat (i1 true), i32 [[TMP9]]) +; NO-VP-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; NO-VP-NEXT: [[TMP7:%.*]] = zext i32 [[INDEX]] to i64 +; NO-VP-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 4 +; NO-VP-NEXT: [[TMP9:%.*]] = getelementptr nuw i8, ptr [[A]], i64 [[TMP8]] +; NO-VP-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64( [[VEC_IND]], ptr align 4 [[TMP9]], i64 16, splat (i1 true), i32 [[TMP3]]) ; NO-VP-NEXT: [[TMP10:%.*]] = getelementptr nuw i8, ptr [[TMP4]], i64 [[TMP8]] -; NO-VP-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64( [[VEC_IND4]], ptr align 4 [[TMP10]], i64 16, splat (i1 true), i32 [[TMP9]]) +; NO-VP-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64( [[VEC_IND]], ptr align 4 [[TMP10]], i64 16, splat (i1 true), i32 [[TMP3]]) ; NO-VP-NEXT: [[TMP11:%.*]] = getelementptr nuw i8, ptr [[TMP5]], i64 [[TMP8]] -; NO-VP-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64( [[VEC_IND4]], ptr align 4 [[TMP11]], i64 16, splat (i1 true), i32 [[TMP9]]) -; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP9]] -; NO-VP-NEXT: [[VEC_IND_NEXT5]] = add nuw nsw [[VEC_IND4]], [[BROADCAST_SPLAT3]] -; NO-VP-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; NO-VP-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] +; NO-VP-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64( [[VEC_IND]], ptr align 4 [[TMP11]], i64 16, splat (i1 true), i32 [[TMP3]]) +; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]] +; NO-VP-NEXT: [[VEC_IND_NEXT]] = add nuw nsw [[VEC_IND]], [[BROADCAST_SPLAT]] +; NO-VP-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] +; NO-VP-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] ; NO-VP: middle.block: ; NO-VP-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]] ; NO-VP-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] @@ -305,14 +305,14 @@ define void @store_factor_4_with_gap(i32 %n, ptr noalias %a) { ; NO-VP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] ; NO-VP-NEXT: br label [[FOR_BODY:%.*]] ; NO-VP: for.body: -; NO-VP-NEXT: [[TMP15:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] -; NO-VP-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP15]], i32 0 -; NO-VP-NEXT: store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4 -; NO-VP-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP15]], i32 1 -; NO-VP-NEXT: store i32 [[TMP15]], ptr [[ARRAYIDX1]], align 4 -; NO-VP-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP15]], i32 3 -; NO-VP-NEXT: store i32 [[TMP15]], ptr [[ARRAYIDX2]], align 4 -; NO-VP-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[TMP15]], 1 +; NO-VP-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] +; NO-VP-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[IV]], i32 0 +; NO-VP-NEXT: store i32 [[IV]], ptr [[ARRAYIDX]], align 4 +; NO-VP-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[IV]], i32 1 +; NO-VP-NEXT: store i32 [[IV]], ptr [[ARRAYIDX1]], align 4 +; NO-VP-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[IV]], i32 3 +; NO-VP-NEXT: store i32 [[IV]], ptr [[ARRAYIDX2]], align 4 +; NO-VP-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 ; NO-VP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]] ; NO-VP-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] ; NO-VP: exit: @@ -353,35 +353,35 @@ define i32 @load_factor_4_with_tail_gap(i64 %n, ptr noalias %a) { ; IF-EVL-NEXT: entry: ; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]] ; IF-EVL: vector.ph: -; IF-EVL-NEXT: [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[A:%.*]], i64 4 -; IF-EVL-NEXT: [[SCEVGEP1:%.*]] = getelementptr nuw i8, ptr [[A]], i64 8 +; IF-EVL-NEXT: [[TMP0:%.*]] = getelementptr nuw i8, ptr [[A:%.*]], i64 4 +; IF-EVL-NEXT: [[TMP1:%.*]] = getelementptr nuw i8, ptr [[A]], i64 8 ; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]] ; IF-EVL: vector.body: ; IF-EVL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ] -; IF-EVL-NEXT: [[VEC_PHI:%.*]] = phi [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[VEC_PHI:%.*]] = phi [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ] ; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ] -; IF-EVL-NEXT: [[TMP4:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) -; IF-EVL-NEXT: [[TMP1:%.*]] = shl nuw i64 [[INDEX]], 4 -; IF-EVL-NEXT: [[TMP2:%.*]] = getelementptr nuw i8, ptr [[A]], i64 [[TMP1]] -; IF-EVL-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP2]], i64 16, splat (i1 true), i32 [[TMP4]]) -; IF-EVL-NEXT: [[TMP8:%.*]] = add [[VEC_PHI]], [[WIDE_MASKED_GATHER]] -; IF-EVL-NEXT: [[TMP6:%.*]] = getelementptr nuw i8, ptr [[SCEVGEP]], i64 [[TMP1]] -; IF-EVL-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP6]], i64 16, splat (i1 true), i32 [[TMP4]]) -; IF-EVL-NEXT: [[TMP9:%.*]] = add [[TMP8]], [[WIDE_MASKED_GATHER1]] -; IF-EVL-NEXT: [[TMP10:%.*]] = getelementptr nuw i8, ptr [[SCEVGEP1]], i64 [[TMP1]] -; IF-EVL-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP10]], i64 16, splat (i1 true), i32 [[TMP4]]) -; IF-EVL-NEXT: [[TMP11:%.*]] = add [[TMP9]], [[WIDE_MASKED_GATHER2]] -; IF-EVL-NEXT: [[TMP12]] = call @llvm.vp.merge.nxv4i32( splat (i1 true), [[TMP11]], [[VEC_PHI]], i32 [[TMP4]]) -; IF-EVL-NEXT: [[TMP5:%.*]] = zext i32 [[TMP4]] to i64 -; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP5]], [[INDEX]] -; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP5]] -; IF-EVL-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 -; IF-EVL-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] +; IF-EVL-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) +; IF-EVL-NEXT: [[TMP3:%.*]] = shl nuw i64 [[INDEX]], 4 +; IF-EVL-NEXT: [[TMP4:%.*]] = getelementptr nuw i8, ptr [[A]], i64 [[TMP3]] +; IF-EVL-NEXT: [[TMP5:%.*]] = call @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP4]], i64 16, splat (i1 true), i32 [[TMP2]]) +; IF-EVL-NEXT: [[TMP6:%.*]] = add [[VEC_PHI]], [[TMP5]] +; IF-EVL-NEXT: [[TMP7:%.*]] = getelementptr nuw i8, ptr [[TMP0]], i64 [[TMP3]] +; IF-EVL-NEXT: [[TMP8:%.*]] = call @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP7]], i64 16, splat (i1 true), i32 [[TMP2]]) +; IF-EVL-NEXT: [[TMP9:%.*]] = add [[TMP6]], [[TMP8]] +; IF-EVL-NEXT: [[TMP10:%.*]] = getelementptr nuw i8, ptr [[TMP1]], i64 [[TMP3]] +; IF-EVL-NEXT: [[TMP11:%.*]] = call @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP10]], i64 16, splat (i1 true), i32 [[TMP2]]) +; IF-EVL-NEXT: [[TMP12:%.*]] = add [[TMP9]], [[TMP11]] +; IF-EVL-NEXT: [[TMP13]] = call @llvm.vp.merge.nxv4i32( splat (i1 true), [[TMP12]], [[VEC_PHI]], i32 [[TMP2]]) +; IF-EVL-NEXT: [[TMP14:%.*]] = zext i32 [[TMP2]] to i64 +; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP14]], [[INDEX]] +; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP14]] +; IF-EVL-NEXT: [[TMP15:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 +; IF-EVL-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; IF-EVL: middle.block: -; IF-EVL-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32( [[TMP12]]) -; IF-EVL-NEXT: br label [[FOR_BODY:%.*]] +; IF-EVL-NEXT: [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32( [[TMP13]]) +; IF-EVL-NEXT: br label [[EXIT:%.*]] ; IF-EVL: exit: -; IF-EVL-NEXT: ret i32 [[TMP15]] +; IF-EVL-NEXT: ret i32 [[TMP16]] ; ; NO-VP-LABEL: @load_factor_4_with_tail_gap( ; NO-VP-NEXT: entry: @@ -390,46 +390,46 @@ define i32 @load_factor_4_with_tail_gap(i64 %n, ptr noalias %a) { ; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[N:%.*]], [[TMP1]] ; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; NO-VP: vector.ph: -; NO-VP-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2 -; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] -; NO-VP-NEXT: [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 -; NO-VP-NEXT: [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP3]], i64 [[N_MOD_VF]] -; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP5]] +; NO-VP-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 2 +; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]] +; NO-VP-NEXT: [[TMP3:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 +; NO-VP-NEXT: [[TMP4:%.*]] = select i1 [[TMP3]], i64 [[TMP2]], i64 [[N_MOD_VF]] +; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP4]] ; NO-VP-NEXT: br label [[VECTOR_BODY:%.*]] ; NO-VP: vector.body: ; NO-VP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; NO-VP-NEXT: [[VEC_PHI:%.*]] = phi [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ] -; NO-VP-NEXT: [[TMP6:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], i64 [[INDEX]], i32 0 -; NO-VP-NEXT: [[WIDE_VEC:%.*]] = load , ptr [[TMP6]], align 4 +; NO-VP-NEXT: [[VEC_PHI:%.*]] = phi [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ] +; NO-VP-NEXT: [[TMP5:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], i64 [[INDEX]], i32 0 +; NO-VP-NEXT: [[WIDE_VEC:%.*]] = load , ptr [[TMP5]], align 4 ; NO-VP-NEXT: [[STRIDED_VEC:%.*]] = call { , , , } @llvm.vector.deinterleave4.nxv16i32( [[WIDE_VEC]]) -; NO-VP-NEXT: [[WIDE_MASKED_GATHER:%.*]] = extractvalue { , , , } [[STRIDED_VEC]], 0 -; NO-VP-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = extractvalue { , , , } [[STRIDED_VEC]], 1 -; NO-VP-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = extractvalue { , , , } [[STRIDED_VEC]], 2 -; NO-VP-NEXT: [[TMP10:%.*]] = add [[VEC_PHI]], [[WIDE_MASKED_GATHER]] -; NO-VP-NEXT: [[TMP11:%.*]] = add [[TMP10]], [[WIDE_MASKED_GATHER1]] -; NO-VP-NEXT: [[TMP13]] = add [[TMP11]], [[WIDE_MASKED_GATHER2]] -; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]] -; NO-VP-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; NO-VP-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] +; NO-VP-NEXT: [[TMP6:%.*]] = extractvalue { , , , } [[STRIDED_VEC]], 0 +; NO-VP-NEXT: [[TMP7:%.*]] = extractvalue { , , , } [[STRIDED_VEC]], 1 +; NO-VP-NEXT: [[TMP8:%.*]] = extractvalue { , , , } [[STRIDED_VEC]], 2 +; NO-VP-NEXT: [[TMP9:%.*]] = add [[VEC_PHI]], [[TMP6]] +; NO-VP-NEXT: [[TMP10:%.*]] = add [[TMP9]], [[TMP7]] +; NO-VP-NEXT: [[TMP11]] = add [[TMP10]], [[TMP8]] +; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]] +; NO-VP-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; NO-VP-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] ; NO-VP: middle.block: -; NO-VP-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32( [[TMP13]]) +; NO-VP-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32( [[TMP11]]) ; NO-VP-NEXT: br label [[SCALAR_PH]] ; NO-VP: scalar.ph: ; NO-VP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] -; NO-VP-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP15]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] +; NO-VP-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP13]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] ; NO-VP-NEXT: br label [[FOR_BODY:%.*]] ; NO-VP: for.body: ; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] ; NO-VP-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD2:%.*]], [[FOR_BODY]] ] ; NO-VP-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 0 -; NO-VP-NEXT: [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 -; NO-VP-NEXT: [[ADD:%.*]] = add nsw i32 [[RDX]], [[TMP16]] +; NO-VP-NEXT: [[TMP14:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +; NO-VP-NEXT: [[ADD:%.*]] = add nsw i32 [[RDX]], [[TMP14]] ; NO-VP-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 1 -; NO-VP-NEXT: [[TMP17:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4 -; NO-VP-NEXT: [[ADD1:%.*]] = add nsw i32 [[ADD]], [[TMP17]] +; NO-VP-NEXT: [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4 +; NO-VP-NEXT: [[ADD1:%.*]] = add nsw i32 [[ADD]], [[TMP15]] ; NO-VP-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 2 -; NO-VP-NEXT: [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4 -; NO-VP-NEXT: [[ADD2]] = add nsw i32 [[ADD1]], [[TMP18]] +; NO-VP-NEXT: [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4 +; NO-VP-NEXT: [[ADD2]] = add nsw i32 [[ADD1]], [[TMP16]] ; NO-VP-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 ; NO-VP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] ; NO-VP-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] @@ -478,30 +478,30 @@ define void @store_factor_4_with_tail_gap(i32 %n, ptr noalias %a) { ; IF-EVL: vector.ph: ; IF-EVL-NEXT: [[TMP0:%.*]] = getelementptr nuw i8, ptr [[A:%.*]], i64 4 ; IF-EVL-NEXT: [[TMP1:%.*]] = getelementptr nuw i8, ptr [[A]], i64 8 -; IF-EVL-NEXT: [[TMP4:%.*]] = call @llvm.stepvector.nxv4i32() +; IF-EVL-NEXT: [[TMP2:%.*]] = call @llvm.stepvector.nxv4i32() ; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]] ; IF-EVL: vector.body: ; IF-EVL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ] -; IF-EVL-NEXT: [[VEC_IND2:%.*]] = phi [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT5:%.*]], [[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP2]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] ; IF-EVL-NEXT: [[AVL:%.*]] = phi i32 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ] -; IF-EVL-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true) -; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[TMP6]], i64 0 +; IF-EVL-NEXT: [[TMP3:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true) +; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[TMP3]], i64 0 ; IF-EVL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer -; IF-EVL-NEXT: [[TMP9:%.*]] = zext i32 [[INDEX]] to i64 -; IF-EVL-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP9]], 4 -; IF-EVL-NEXT: [[TMP10:%.*]] = getelementptr nuw i8, ptr [[A]], i64 [[TMP5]] -; IF-EVL-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64( [[VEC_IND2]], ptr align 4 [[TMP10]], i64 16, splat (i1 true), i32 [[TMP6]]) +; IF-EVL-NEXT: [[TMP4:%.*]] = zext i32 [[INDEX]] to i64 +; IF-EVL-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 4 +; IF-EVL-NEXT: [[TMP6:%.*]] = getelementptr nuw i8, ptr [[A]], i64 [[TMP5]] +; IF-EVL-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64( [[VEC_IND]], ptr align 4 [[TMP6]], i64 16, splat (i1 true), i32 [[TMP3]]) ; IF-EVL-NEXT: [[TMP7:%.*]] = getelementptr nuw i8, ptr [[TMP0]], i64 [[TMP5]] -; IF-EVL-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64( [[VEC_IND2]], ptr align 4 [[TMP7]], i64 16, splat (i1 true), i32 [[TMP6]]) -; IF-EVL-NEXT: [[TMP11:%.*]] = getelementptr nuw i8, ptr [[TMP1]], i64 [[TMP5]] -; IF-EVL-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64( [[VEC_IND2]], ptr align 4 [[TMP11]], i64 16, splat (i1 true), i32 [[TMP6]]) -; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add i32 [[TMP6]], [[INDEX]] -; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP6]] -; IF-EVL-NEXT: [[VEC_IND_NEXT5]] = add nuw nsw [[VEC_IND2]], [[BROADCAST_SPLAT]] -; IF-EVL-NEXT: [[TMP8:%.*]] = icmp eq i32 [[AVL_NEXT]], 0 -; IF-EVL-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] +; IF-EVL-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64( [[VEC_IND]], ptr align 4 [[TMP7]], i64 16, splat (i1 true), i32 [[TMP3]]) +; IF-EVL-NEXT: [[TMP8:%.*]] = getelementptr nuw i8, ptr [[TMP1]], i64 [[TMP5]] +; IF-EVL-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64( [[VEC_IND]], ptr align 4 [[TMP8]], i64 16, splat (i1 true), i32 [[TMP3]]) +; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add i32 [[TMP3]], [[INDEX]] +; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP3]] +; IF-EVL-NEXT: [[VEC_IND_NEXT]] = add nuw nsw [[VEC_IND]], [[BROADCAST_SPLAT]] +; IF-EVL-NEXT: [[TMP9:%.*]] = icmp eq i32 [[AVL_NEXT]], 0 +; IF-EVL-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] ; IF-EVL: middle.block: -; IF-EVL-NEXT: br label [[FOR_BODY:%.*]] +; IF-EVL-NEXT: br label [[EXIT:%.*]] ; IF-EVL: exit: ; IF-EVL-NEXT: ret void ; @@ -535,8 +535,8 @@ define void @store_factor_4_with_tail_gap(i32 %n, ptr noalias %a) { ; NO-VP-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64( [[VEC_IND]], ptr align 4 [[TMP11]], i64 16, splat (i1 true), i32 [[TMP3]]) ; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]] ; NO-VP-NEXT: [[VEC_IND_NEXT]] = add nuw nsw [[VEC_IND]], [[BROADCAST_SPLAT]] -; NO-VP-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; NO-VP-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] +; NO-VP-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] +; NO-VP-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] ; NO-VP: middle.block: ; NO-VP-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]] ; NO-VP-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] @@ -544,14 +544,14 @@ define void @store_factor_4_with_tail_gap(i32 %n, ptr noalias %a) { ; NO-VP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] ; NO-VP-NEXT: br label [[FOR_BODY:%.*]] ; NO-VP: for.body: -; NO-VP-NEXT: [[TMP15:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] -; NO-VP-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP15]], i32 0 -; NO-VP-NEXT: store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4 -; NO-VP-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP15]], i32 1 -; NO-VP-NEXT: store i32 [[TMP15]], ptr [[ARRAYIDX1]], align 4 -; NO-VP-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP15]], i32 2 -; NO-VP-NEXT: store i32 [[TMP15]], ptr [[ARRAYIDX2]], align 4 -; NO-VP-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[TMP15]], 1 +; NO-VP-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] +; NO-VP-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[IV]], i32 0 +; NO-VP-NEXT: store i32 [[IV]], ptr [[ARRAYIDX]], align 4 +; NO-VP-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[IV]], i32 1 +; NO-VP-NEXT: store i32 [[IV]], ptr [[ARRAYIDX1]], align 4 +; NO-VP-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[IV]], i32 2 +; NO-VP-NEXT: store i32 [[IV]], ptr [[ARRAYIDX2]], align 4 +; NO-VP-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 ; NO-VP-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]] ; NO-VP-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] ; NO-VP: exit: @@ -579,122 +579,122 @@ exit: define i32 @load_factor_4_reverse(i64 %n, ptr noalias %a) { ; IF-EVL-LABEL: @load_factor_4_reverse( ; IF-EVL-NEXT: entry: -; IF-EVL-NEXT: [[TMP20:%.*]] = add nsw i64 [[N:%.*]], -1 -; IF-EVL-NEXT: [[TMP21:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP20]], i64 0) -; IF-EVL-NEXT: [[TMP1:%.*]] = sub i64 [[N]], [[TMP21]] +; IF-EVL-NEXT: [[TMP0:%.*]] = add nsw i64 [[N:%.*]], -1 +; IF-EVL-NEXT: [[TMP1:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP0]], i64 0) +; IF-EVL-NEXT: [[TMP2:%.*]] = sub i64 [[N]], [[TMP1]] ; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]] ; IF-EVL: vector.ph: ; IF-EVL-NEXT: [[TMP3:%.*]] = shl i64 [[N]], 4 -; IF-EVL-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP3]] +; IF-EVL-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP3]] ; IF-EVL-NEXT: [[TMP5:%.*]] = add nuw nsw i64 [[TMP3]], 4 -; IF-EVL-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]] -; IF-EVL-NEXT: [[TMP22:%.*]] = add nuw nsw i64 [[TMP3]], 8 -; IF-EVL-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP22]] -; IF-EVL-NEXT: [[TMP23:%.*]] = add nuw nsw i64 [[TMP3]], 12 -; IF-EVL-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP23]] +; IF-EVL-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]] +; IF-EVL-NEXT: [[TMP7:%.*]] = add nuw nsw i64 [[TMP3]], 8 +; IF-EVL-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP7]] +; IF-EVL-NEXT: [[TMP9:%.*]] = add nuw nsw i64 [[TMP3]], 12 +; IF-EVL-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP9]] ; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]] ; IF-EVL: vector.body: ; IF-EVL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ] -; IF-EVL-NEXT: [[VEC_PHI:%.*]] = phi [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ] -; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP1]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ] -; IF-EVL-NEXT: [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) -; IF-EVL-NEXT: [[TMP9:%.*]] = mul i64 [[INDEX]], -16 -; IF-EVL-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP9]] -; IF-EVL-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP8]], i64 -16, splat (i1 true), i32 [[TMP6]]) -; IF-EVL-NEXT: [[TMP10:%.*]] = add [[VEC_PHI]], [[WIDE_MASKED_GATHER]] -; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SCEVGEP3]], i64 [[TMP9]] -; IF-EVL-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP12]], i64 -16, splat (i1 true), i32 [[TMP6]]) -; IF-EVL-NEXT: [[TMP11:%.*]] = add [[TMP10]], [[WIDE_MASKED_GATHER3]] -; IF-EVL-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[SCEVGEP2]], i64 [[TMP9]] -; IF-EVL-NEXT: [[WIDE_MASKED_GATHER4:%.*]] = call @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP14]], i64 -16, splat (i1 true), i32 [[TMP6]]) -; IF-EVL-NEXT: [[TMP13:%.*]] = add [[TMP11]], [[WIDE_MASKED_GATHER4]] -; IF-EVL-NEXT: [[TMP17:%.*]] = getelementptr i8, ptr [[SCEVGEP1]], i64 [[TMP9]] -; IF-EVL-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP17]], i64 -16, splat (i1 true), i32 [[TMP6]]) -; IF-EVL-NEXT: [[TMP15:%.*]] = add [[TMP13]], [[WIDE_MASKED_GATHER5]] -; IF-EVL-NEXT: [[TMP16]] = call @llvm.vp.merge.nxv4i32( splat (i1 true), [[TMP15]], [[VEC_PHI]], i32 [[TMP6]]) -; IF-EVL-NEXT: [[TMP7:%.*]] = zext i32 [[TMP6]] to i64 -; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP7]], [[INDEX]] -; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]] -; IF-EVL-NEXT: [[TMP18:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 -; IF-EVL-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] +; IF-EVL-NEXT: [[VEC_PHI:%.*]] = phi [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP25:%.*]], [[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP2]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true) +; IF-EVL-NEXT: [[TMP12:%.*]] = mul i64 [[INDEX]], -16 +; IF-EVL-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[TMP4]], i64 [[TMP12]] +; IF-EVL-NEXT: [[TMP14:%.*]] = call @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP13]], i64 -16, splat (i1 true), i32 [[TMP11]]) +; IF-EVL-NEXT: [[TMP15:%.*]] = add [[VEC_PHI]], [[TMP14]] +; IF-EVL-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[TMP6]], i64 [[TMP12]] +; IF-EVL-NEXT: [[TMP17:%.*]] = call @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP16]], i64 -16, splat (i1 true), i32 [[TMP11]]) +; IF-EVL-NEXT: [[TMP18:%.*]] = add [[TMP15]], [[TMP17]] +; IF-EVL-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[TMP8]], i64 [[TMP12]] +; IF-EVL-NEXT: [[TMP20:%.*]] = call @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP19]], i64 -16, splat (i1 true), i32 [[TMP11]]) +; IF-EVL-NEXT: [[TMP21:%.*]] = add [[TMP18]], [[TMP20]] +; IF-EVL-NEXT: [[TMP22:%.*]] = getelementptr i8, ptr [[TMP10]], i64 [[TMP12]] +; IF-EVL-NEXT: [[TMP23:%.*]] = call @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP22]], i64 -16, splat (i1 true), i32 [[TMP11]]) +; IF-EVL-NEXT: [[TMP24:%.*]] = add [[TMP21]], [[TMP23]] +; IF-EVL-NEXT: [[TMP25]] = call @llvm.vp.merge.nxv4i32( splat (i1 true), [[TMP24]], [[VEC_PHI]], i32 [[TMP11]]) +; IF-EVL-NEXT: [[TMP26:%.*]] = zext i32 [[TMP11]] to i64 +; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP26]], [[INDEX]] +; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP26]] +; IF-EVL-NEXT: [[TMP27:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 +; IF-EVL-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] ; IF-EVL: middle.block: -; IF-EVL-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32( [[TMP16]]) -; IF-EVL-NEXT: br label [[FOR_BODY:%.*]] +; IF-EVL-NEXT: [[TMP28:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32( [[TMP25]]) +; IF-EVL-NEXT: br label [[EXIT:%.*]] ; IF-EVL: exit: -; IF-EVL-NEXT: ret i32 [[TMP19]] +; IF-EVL-NEXT: ret i32 [[TMP28]] ; ; NO-VP-LABEL: @load_factor_4_reverse( ; NO-VP-NEXT: entry: -; NO-VP-NEXT: [[TMP27:%.*]] = add nsw i64 [[N:%.*]], -1 -; NO-VP-NEXT: [[TMP28:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP27]], i64 0) -; NO-VP-NEXT: [[TMP1:%.*]] = sub i64 [[N]], [[TMP28]] -; NO-VP-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() -; NO-VP-NEXT: [[TMP29:%.*]] = shl nuw i64 [[TMP4]], 2 -; NO-VP-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP29]], i64 8) -; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], [[UMAX]] +; NO-VP-NEXT: [[TMP0:%.*]] = add nsw i64 [[N:%.*]], -1 +; NO-VP-NEXT: [[TMP1:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP0]], i64 0) +; NO-VP-NEXT: [[TMP2:%.*]] = sub i64 [[N]], [[TMP1]] +; NO-VP-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() +; NO-VP-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 2 +; NO-VP-NEXT: [[TMP5:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP4]], i64 8) +; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP5]] ; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; NO-VP: vector.ph: -; NO-VP-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 2 -; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP5]] -; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]] -; NO-VP-NEXT: [[TMP6:%.*]] = sub i64 [[N]], [[N_VEC]] -; NO-VP-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP5]] to i32 +; NO-VP-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP3]], 2 +; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]] +; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] +; NO-VP-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[N_VEC]] +; NO-VP-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP6]] to i32 ; NO-VP-NEXT: [[TMP9:%.*]] = shl i64 [[N]], 4 -; NO-VP-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP9]] -; NO-VP-NEXT: [[TMP30:%.*]] = add nuw nsw i64 [[TMP9]], 4 -; NO-VP-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP30]] -; NO-VP-NEXT: [[TMP31:%.*]] = add nuw nsw i64 [[TMP9]], 8 -; NO-VP-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP31]] -; NO-VP-NEXT: [[TMP32:%.*]] = add nuw nsw i64 [[TMP9]], 12 -; NO-VP-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP32]] +; NO-VP-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP9]] +; NO-VP-NEXT: [[TMP11:%.*]] = add nuw nsw i64 [[TMP9]], 4 +; NO-VP-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP11]] +; NO-VP-NEXT: [[TMP13:%.*]] = add nuw nsw i64 [[TMP9]], 8 +; NO-VP-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP13]] +; NO-VP-NEXT: [[TMP15:%.*]] = add nuw nsw i64 [[TMP9]], 12 +; NO-VP-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP15]] ; NO-VP-NEXT: br label [[VECTOR_BODY:%.*]] ; NO-VP: vector.body: ; NO-VP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; NO-VP-NEXT: [[VEC_PHI:%.*]] = phi [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ] -; NO-VP-NEXT: [[TMP23:%.*]] = mul i64 [[INDEX]], -16 -; NO-VP-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[TMP23]] -; NO-VP-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP13]], i64 -16, splat (i1 true), i32 [[TMP15]]) -; NO-VP-NEXT: [[TMP11:%.*]] = add [[VEC_PHI]], [[WIDE_MASKED_GATHER]] -; NO-VP-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[SCEVGEP3]], i64 [[TMP23]] -; NO-VP-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP24]], i64 -16, splat (i1 true), i32 [[TMP15]]) -; NO-VP-NEXT: [[TMP12:%.*]] = add [[TMP11]], [[WIDE_MASKED_GATHER3]] -; NO-VP-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[SCEVGEP2]], i64 [[TMP23]] -; NO-VP-NEXT: [[WIDE_MASKED_GATHER4:%.*]] = call @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP25]], i64 -16, splat (i1 true), i32 [[TMP15]]) -; NO-VP-NEXT: [[TMP14:%.*]] = add [[TMP12]], [[WIDE_MASKED_GATHER4]] -; NO-VP-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[SCEVGEP1]], i64 [[TMP23]] -; NO-VP-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP26]], i64 -16, splat (i1 true), i32 [[TMP15]]) -; NO-VP-NEXT: [[TMP16]] = add [[TMP14]], [[WIDE_MASKED_GATHER5]] -; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] -; NO-VP-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; NO-VP-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] +; NO-VP-NEXT: [[VEC_PHI:%.*]] = phi [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP29:%.*]], [[VECTOR_BODY]] ] +; NO-VP-NEXT: [[TMP17:%.*]] = mul i64 [[INDEX]], -16 +; NO-VP-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[TMP10]], i64 [[TMP17]] +; NO-VP-NEXT: [[TMP19:%.*]] = call @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP18]], i64 -16, splat (i1 true), i32 [[TMP8]]) +; NO-VP-NEXT: [[TMP20:%.*]] = add [[VEC_PHI]], [[TMP19]] +; NO-VP-NEXT: [[TMP21:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP17]] +; NO-VP-NEXT: [[TMP22:%.*]] = call @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP21]], i64 -16, splat (i1 true), i32 [[TMP8]]) +; NO-VP-NEXT: [[TMP23:%.*]] = add [[TMP20]], [[TMP22]] +; NO-VP-NEXT: [[TMP24:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP17]] +; NO-VP-NEXT: [[TMP25:%.*]] = call @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP24]], i64 -16, splat (i1 true), i32 [[TMP8]]) +; NO-VP-NEXT: [[TMP26:%.*]] = add [[TMP23]], [[TMP25]] +; NO-VP-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[TMP16]], i64 [[TMP17]] +; NO-VP-NEXT: [[TMP28:%.*]] = call @llvm.experimental.vp.strided.load.nxv4i32.p0.i64(ptr align 4 [[TMP27]], i64 -16, splat (i1 true), i32 [[TMP8]]) +; NO-VP-NEXT: [[TMP29]] = add [[TMP26]], [[TMP28]] +; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]] +; NO-VP-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; NO-VP-NEXT: br i1 [[TMP30]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] ; NO-VP: middle.block: -; NO-VP-NEXT: [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32( [[TMP16]]) -; NO-VP-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]] +; NO-VP-NEXT: [[TMP31:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32( [[TMP29]]) +; NO-VP-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] ; NO-VP-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] ; NO-VP: scalar.ph: -; NO-VP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ [[N]], [[ENTRY:%.*]] ] -; NO-VP-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP18]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] +; NO-VP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ [[N]], [[ENTRY:%.*]] ] +; NO-VP-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP31]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] ; NO-VP-NEXT: br label [[FOR_BODY:%.*]] ; NO-VP: for.body: ; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] ; NO-VP-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD3:%.*]], [[FOR_BODY]] ] ; NO-VP-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 0 -; NO-VP-NEXT: [[TMP19:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 -; NO-VP-NEXT: [[ADD:%.*]] = add nsw i32 [[RDX]], [[TMP19]] +; NO-VP-NEXT: [[TMP32:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +; NO-VP-NEXT: [[ADD:%.*]] = add nsw i32 [[RDX]], [[TMP32]] ; NO-VP-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 1 -; NO-VP-NEXT: [[TMP20:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4 -; NO-VP-NEXT: [[ADD1:%.*]] = add nsw i32 [[ADD]], [[TMP20]] +; NO-VP-NEXT: [[TMP33:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4 +; NO-VP-NEXT: [[ADD1:%.*]] = add nsw i32 [[ADD]], [[TMP33]] ; NO-VP-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 2 -; NO-VP-NEXT: [[TMP21:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4 -; NO-VP-NEXT: [[ADD2:%.*]] = add nsw i32 [[ADD1]], [[TMP21]] +; NO-VP-NEXT: [[TMP34:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4 +; NO-VP-NEXT: [[ADD2:%.*]] = add nsw i32 [[ADD1]], [[TMP34]] ; NO-VP-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 3 -; NO-VP-NEXT: [[TMP22:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4 -; NO-VP-NEXT: [[ADD3]] = add nsw i32 [[ADD2]], [[TMP22]] +; NO-VP-NEXT: [[TMP35:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4 +; NO-VP-NEXT: [[ADD3]] = add nsw i32 [[ADD2]], [[TMP35]] ; NO-VP-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], -1 ; NO-VP-NEXT: [[EXITCOND:%.*]] = icmp sgt i64 [[IV_NEXT]], 0 ; NO-VP-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[EXIT]], !llvm.loop [[LOOP13:![0-9]+]] ; NO-VP: exit: -; NO-VP-NEXT: [[ADD3_LCSSA:%.*]] = phi i32 [ [[ADD3]], [[FOR_BODY]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ] +; NO-VP-NEXT: [[ADD3_LCSSA:%.*]] = phi i32 [ [[ADD3]], [[FOR_BODY]] ], [ [[TMP31]], [[MIDDLE_BLOCK]] ] ; NO-VP-NEXT: ret i32 [[ADD3_LCSSA]] ; entry: @@ -729,7 +729,7 @@ define void @store_factor_4_reverse(i32 %n, ptr noalias %a) { ; IF-EVL-NEXT: [[TMP0:%.*]] = add nsw i32 [[N:%.*]], -1 ; IF-EVL-NEXT: [[TMP1:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP0]], i32 0) ; IF-EVL-NEXT: [[TMP2:%.*]] = sub i32 [[N]], [[TMP1]] -; IF-EVL-NEXT: br label [[FOR_BODY:%.*]] +; IF-EVL-NEXT: br label [[VECTOR_PH:%.*]] ; IF-EVL: vector.ph: ; IF-EVL-NEXT: [[TMP3:%.*]] = sext i32 [[N]] to i64 ; IF-EVL-NEXT: [[TMP4:%.*]] = shl nsw i64 [[TMP3]], 4 @@ -746,9 +746,9 @@ define void @store_factor_4_reverse(i32 %n, ptr noalias %a) { ; IF-EVL-NEXT: [[TMP13:%.*]] = sub nsw [[BROADCAST_SPLAT]], [[TMP12]] ; IF-EVL-NEXT: br label [[VECTOR_BODY:%.*]] ; IF-EVL: vector.body: -; IF-EVL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[FOR_BODY]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ] -; IF-EVL-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP13]], [[FOR_BODY]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] -; IF-EVL-NEXT: [[AVL:%.*]] = phi i32 [ [[TMP2]], [[FOR_BODY]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], [[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP13]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; IF-EVL-NEXT: [[AVL:%.*]] = phi i32 [ [[TMP2]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ] ; IF-EVL-NEXT: [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true) ; IF-EVL-NEXT: [[TMP15:%.*]] = sub nsw i32 0, [[TMP14]] ; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement poison, i32 [[TMP15]], i64 0 @@ -804,12 +804,12 @@ define void @store_factor_4_reverse(i32 %n, ptr noalias %a) { ; NO-VP-NEXT: [[TMP19:%.*]] = sub nsw i32 0, [[TMP6]] ; NO-VP-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement poison, i32 [[TMP19]], i64 0 ; NO-VP-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector [[BROADCAST_SPLATINSERT1]], poison, zeroinitializer -; NO-VP-NEXT: br label [[FOR_BODY:%.*]] +; NO-VP-NEXT: br label [[VECTOR_BODY:%.*]] ; NO-VP: vector.body: -; NO-VP-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR_BODY]] ] -; NO-VP-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP18]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR_BODY]] ] -; NO-VP-NEXT: [[TMP27:%.*]] = zext i32 [[INDEX]] to i64 -; NO-VP-NEXT: [[TMP21:%.*]] = mul i64 [[TMP27]], -16 +; NO-VP-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; NO-VP-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP18]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; NO-VP-NEXT: [[TMP20:%.*]] = zext i32 [[INDEX]] to i64 +; NO-VP-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], -16 ; NO-VP-NEXT: [[TMP22:%.*]] = getelementptr i8, ptr [[TMP10]], i64 [[TMP21]] ; NO-VP-NEXT: call void @llvm.experimental.vp.strided.store.nxv4i32.p0.i64( [[VEC_IND]], ptr align 4 [[TMP22]], i64 -16, splat (i1 true), i32 [[TMP6]]) ; NO-VP-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP21]] @@ -821,26 +821,26 @@ define void @store_factor_4_reverse(i32 %n, ptr noalias %a) { ; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP6]] ; NO-VP-NEXT: [[VEC_IND_NEXT]] = add nsw [[VEC_IND]], [[BROADCAST_SPLAT2]] ; NO-VP-NEXT: [[TMP26:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; NO-VP-NEXT: br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] +; NO-VP-NEXT: br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] ; NO-VP: middle.block: ; NO-VP-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]] ; NO-VP-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] ; NO-VP: scalar.ph: ; NO-VP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ [[N]], [[ENTRY:%.*]] ] -; NO-VP-NEXT: br label [[FOR_BODY1:%.*]] +; NO-VP-NEXT: br label [[FOR_BODY:%.*]] ; NO-VP: for.body: -; NO-VP-NEXT: [[TMP20:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY1]] ] -; NO-VP-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP20]], i32 0 -; NO-VP-NEXT: store i32 [[TMP20]], ptr [[ARRAYIDX]], align 4 -; NO-VP-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP20]], i32 1 -; NO-VP-NEXT: store i32 [[TMP20]], ptr [[ARRAYIDX1]], align 4 -; NO-VP-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP20]], i32 2 -; NO-VP-NEXT: store i32 [[TMP20]], ptr [[ARRAYIDX2]], align 4 -; NO-VP-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP20]], i32 3 -; NO-VP-NEXT: store i32 [[TMP20]], ptr [[ARRAYIDX3]], align 4 -; NO-VP-NEXT: [[IV_NEXT]] = add nsw i32 [[TMP20]], -1 +; NO-VP-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] +; NO-VP-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[IV]], i32 0 +; NO-VP-NEXT: store i32 [[IV]], ptr [[ARRAYIDX]], align 4 +; NO-VP-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[IV]], i32 1 +; NO-VP-NEXT: store i32 [[IV]], ptr [[ARRAYIDX1]], align 4 +; NO-VP-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[IV]], i32 2 +; NO-VP-NEXT: store i32 [[IV]], ptr [[ARRAYIDX2]], align 4 +; NO-VP-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[IV]], i32 3 +; NO-VP-NEXT: store i32 [[IV]], ptr [[ARRAYIDX3]], align 4 +; NO-VP-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], -1 ; NO-VP-NEXT: [[EXITCOND:%.*]] = icmp sgt i32 [[IV_NEXT]], 0 -; NO-VP-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY1]], label [[EXIT]], !llvm.loop [[LOOP15:![0-9]+]] +; NO-VP-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[EXIT]], !llvm.loop [[LOOP15:![0-9]+]] ; NO-VP: exit: ; NO-VP-NEXT: ret void ; diff --git a/llvm/test/Transforms/LoopVectorize/scalable-lifetime.ll b/llvm/test/Transforms/LoopVectorize/scalable-lifetime.ll index be9221e3eaf9b..4cd66a3113620 100644 --- a/llvm/test/Transforms/LoopVectorize/scalable-lifetime.ll +++ b/llvm/test/Transforms/LoopVectorize/scalable-lifetime.ll @@ -1,4 +1,4 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6 ; RUN: opt -S -passes=loop-vectorize -force-vector-width="vscale x 2" -force-vector-interleave=1 < %s | FileCheck %s target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" @@ -6,47 +6,47 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 ; Make sure we can vectorize loops which contain lifetime markers. define void @test(ptr %d) { -; CHECK-LABEL: define void @test -; CHECK-SAME: (ptr [[D:%.*]]) { -; CHECK-NEXT: entry: +; CHECK-LABEL: define void @test( +; CHECK-SAME: ptr [[D:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: ; CHECK-NEXT: [[ARR:%.*]] = alloca [1024 x i32], align 16 ; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[ARR]]) ; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1 ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 128, [[TMP1]] -; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] -; CHECK: vector.ph: -; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1 -; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 128, [[TMP3]] +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 1 +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 128, [[TMP2]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 128, [[N_MOD_VF]] -; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] -; CHECK: vector.body: -; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] ; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[ARR]]) -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[INDEX]] -; CHECK-NEXT: store splat (i32 100), ptr [[TMP4]], align 8 +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[INDEX]] +; CHECK-NEXT: store splat (i32 100), ptr [[TMP3]], align 8 ; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[ARR]]) -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]] -; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] -; CHECK: middle.block: +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]] +; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 128, [[N_VEC]] -; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] -; CHECK: scalar.ph: -; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] -; CHECK-NEXT: br label [[FOR_BODY:%.*]] -; CHECK: for.body: -; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] +; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] +; CHECK-NEXT: br label %[[FOR_BODY:.*]] +; CHECK: [[FOR_BODY]]: +; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ] ; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[ARR]]) ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[INDVARS_IV]] -; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[ARRAYIDX]], align 8 +; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX]], align 8 ; CHECK-NEXT: store i32 100, ptr [[ARRAYIDX]], align 8 ; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[ARR]]) ; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1 ; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 ; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i32 [[LFTR_WIDEIV]], 128 -; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP3:![0-9]+]] -; CHECK: for.end: +; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP3:![0-9]+]] +; CHECK: [[FOR_END]]: ; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[ARR]]) ; CHECK-NEXT: ret void ; @@ -74,47 +74,47 @@ for.end: } define void @testloopvariant(ptr %d) { -; CHECK-LABEL: define void @testloopvariant -; CHECK-SAME: (ptr [[D:%.*]]) { -; CHECK-NEXT: entry: +; CHECK-LABEL: define void @testloopvariant( +; CHECK-SAME: ptr [[D:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: ; CHECK-NEXT: [[ARR:%.*]] = alloca [1024 x i32], align 16 ; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1 ; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 128, [[TMP1]] -; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] -; CHECK: vector.ph: -; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 1 -; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 128, [[TMP3]] +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 1 +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 128, [[TMP2]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 128, [[N_MOD_VF]] -; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] -; CHECK: vector.body: -; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] ; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[ARR]]) -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[INDEX]] -; CHECK-NEXT: store splat (i32 100), ptr [[TMP4]], align 8 +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[INDEX]] +; CHECK-NEXT: store splat (i32 100), ptr [[TMP3]], align 8 ; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[ARR]]) -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]] -; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] -; CHECK: middle.block: +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]] +; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 128, [[N_VEC]] -; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] -; CHECK: scalar.ph: -; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] -; CHECK-NEXT: br label [[FOR_BODY:%.*]] -; CHECK: for.body: -; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr [1024 x i32], ptr [[ARR]], i32 0, i64 [[INDVARS_IV]] +; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] +; CHECK-NEXT: br label %[[FOR_BODY:.*]] +; CHECK: [[FOR_BODY]]: +; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ] +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr [1024 x i32], ptr [[ARR]], i32 0, i64 [[INDVARS_IV]] ; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[ARR]]) ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[INDVARS_IV]] -; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 8 +; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[ARRAYIDX]], align 8 ; CHECK-NEXT: store i32 100, ptr [[ARRAYIDX]], align 8 ; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[ARR]]) ; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1 ; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32 ; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i32 [[LFTR_WIDEIV]], 128 -; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP5:![0-9]+]] -; CHECK: for.end: +; CHECK-NEXT: br i1 [[EXITCOND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP5:![0-9]+]] +; CHECK: [[FOR_END]]: ; CHECK-NEXT: ret void ; entry: