feat: implement eliminate range/null check in loop

llvm/include/llvm/Analysis/LoopAccessAnalysis.h

@@ -42,6 +42,13 @@ struct VectorizerParams {
   /// True if force-vector-interleave was specified by the user.
   LLVM_ABI static bool isInterleaveForced();
 
+  /// Allow the vectorizers to treat unordered, non-volatile primitive atomic
+  /// accesses as vectorizable.  The Java front end represents plain heap
+  /// element stores as unordered atomics until their safepoint/GC lowering is
+  /// complete.  Keep this switch shared by LoopAccessAnalysis, LoopVectorize,
+  /// and SLPVectorizer so that the command-line option has one meaning.
+  LLVM_ABI static bool IgnoreAtomicity;
+
   /// \When performing memory disambiguation checks at runtime do not
   /// make more than this number of comparisons.
   LLVM_ABI static unsigned RuntimeMemoryCheckThreshold;

llvm/include/llvm/Jeandle/Jeandle.h

@@ -18,7 +18,7 @@
 namespace llvm::jeandle {
 
 void optimize(Module &M, OptimizationLevel Level = OptimizationLevel::O3,
-              PipelineMode Mode = PipelineMode::MethodCompilation);
+              PipelineMode Mode = PipelineMode::MethodCompilation, TargetMachine *TM = nullptr);
 
 } // end namespace llvm::jeandle
 

llvm/include/llvm/Jeandle/Pipeline.h

@@ -28,7 +28,7 @@ enum class PipelineMode {
 class Pipeline {
 public:
   Pipeline(OptimizationLevel Level, LLVMContext &Ctx,
-           PipelineMode Mode = PipelineMode::MethodCompilation);
+           PipelineMode Mode = PipelineMode::MethodCompilation, TargetMachine *TM = nullptr);
 
   LLVM_ABI static ModulePassManager
   buildJeandlePipeline(PassBuilder &PB, OptimizationLevel Level,

llvm/lib/Analysis/Jeandle/PartialEscapeAnalysis.cpp

@@ -5107,7 +5107,7 @@ void Analyzer::processInstruction(Instruction *I) {
     // yet:
     //   - TODO: processArrayCopy / processMemSet — llvm.memcpy/memmove
     //     (System.arraycopy) and llvm.memset (Arrays.fill). The only
-    //     llvm.memset producer today is jeandle.new_instance's lower-phase=1
+    //     llvm.memset producer today is jeandle.new_instance's lower-phase=2
     //     template, inlined AFTER PEA, so neither shape reaches PEA yet.
     //   - TODO: llvm.reachability_fence — upstream LLVM this fork tracks
     //     does not define Intrinsic::reachability_fence, and the frontend
@@ -6288,7 +6288,7 @@ bool Analyzer::foldGetClass(CallBase *CB) {
 bool Analyzer::foldCheckCast(CallBase *CB) {
   // jeandle.checkcast itself is lower-phase="0" by design: its expansion
   // exposes a null check (foldICmpEquality) and a jeandle.check_instanceof
-  // call (lower-phase="1"), which is the subtype-check op this fold sees in
+  // call (lower-phase="2"), which is the subtype-check op this fold sees in
   // production. The direct jeandle.checkcast form is only reachable from lit
   // tests.
   if (CB->arg_size() < 2)
@@ -6317,7 +6317,7 @@ bool Analyzer::foldCheckCast(CallBase *CB) {
 bool Analyzer::foldInstanceOf(CallBase *CB) {
   // jeandle.instanceof itself is lower-phase="0" by design: its expansion
   // exposes a null check (foldICmpEquality) and a jeandle.check_instanceof
-  // call (lower-phase="1", handled by foldCheckCast). The direct
+  // call (lower-phase="2", handled by foldCheckCast). The direct
   // jeandle.instanceof form is only reachable from lit tests.
   if (CB->arg_size() < 2)
     return false;

llvm/lib/Analysis/LoopAccessAnalysis.cpp

@@ -148,6 +148,13 @@ static cl::opt<bool, true> HoistRuntimeChecks(
     cl::location(VectorizerParams::HoistRuntimeChecks), cl::init(true));
 bool VectorizerParams::HoistRuntimeChecks;
 
+static cl::opt<bool, true> VectorizerIgnoreAtomicity(
+    "vectorizer-ignore-atomicity",
+    cl::desc("Allow vectorizing atomic unordered loads/stores"),
+    cl::location(VectorizerParams::IgnoreAtomicity), cl::init(false));
+
+bool VectorizerParams::IgnoreAtomicity;
+
 bool VectorizerParams::isInterleaveForced() {
   return ::VectorizationInterleave.getNumOccurrences() > 0;
 }
@@ -2588,7 +2595,9 @@ bool LoopAccessInfo::analyzeLoop(AAResults *AA, const LoopInfo *LI,
           HasComplexMemInst = true;
           continue;
         }
-        if (!Ld->isSimple() && !IsAnnotatedParallel) {
+        if (!Ld->isSimple() && !IsAnnotatedParallel &&
+            !(VectorizerIgnoreAtomicity && Ld->isUnordered() &&
+              !Ld->getType()->isPointerTy())) {
           recordAnalysis("NonSimpleLoad", Ld)
               << "read with atomic ordering or volatile read";
           LLVM_DEBUG(dbgs() << "LAA: Found a non-simple load.\n");
@@ -2612,7 +2621,9 @@ bool LoopAccessInfo::analyzeLoop(AAResults *AA, const LoopInfo *LI,
           HasComplexMemInst = true;
           continue;
         }
-        if (!St->isSimple() && !IsAnnotatedParallel) {
+        if (!St->isSimple() && !IsAnnotatedParallel &&
+            !(VectorizerIgnoreAtomicity && St->isUnordered() &&
+              !St->getValueOperand()->getType()->isPointerTy())) {
           recordAnalysis("NonSimpleStore", St)
               << "write with atomic ordering or volatile write";
           LLVM_DEBUG(dbgs() << "LAA: Found a non-simple store.\n");

llvm/lib/Jeandle/Jeandle.cpp

@@ -13,8 +13,8 @@
 
 namespace llvm::jeandle {
 
-void optimize(Module &M, OptimizationLevel Level, PipelineMode Mode) {
-  Pipeline P(Level, M.getContext(), Mode);
+void optimize(Module &M, OptimizationLevel Level, PipelineMode Mode, llvm::TargetMachine *TM) {
+  Pipeline P(Level, M.getContext(), Mode, TM);
   P.run(M);
 }
 

llvm/lib/Jeandle/Pipeline.cpp

@@ -33,12 +33,16 @@
 #include "llvm/Transforms/Jeandle/TLSPointerRewrite.h"
 #include "llvm/Transforms/Jeandle/TypeCheckElimination.h"
 #include "llvm/Transforms/Scalar/ADCE.h"
+#include "llvm/Transforms/Scalar/ConstraintElimination.h"
+#include "llvm/Transforms/Scalar/DeadStoreElimination.h"
 #include "llvm/Transforms/Scalar/EarlyCSE.h"
 #include "llvm/Transforms/Scalar/GVN.h"
+#include "llvm/Transforms/Scalar/InductiveRangeCheckElimination.h"
 #include "llvm/Transforms/Scalar/IndVarSimplify.h"
 #include "llvm/Transforms/Scalar/InstSimplifyPass.h"
 #include "llvm/Transforms/Scalar/LICM.h"
 #include "llvm/Transforms/Scalar/LoopDeletion.h"
+#include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
 #include "llvm/Transforms/Scalar/LoopPassManager.h"
 #include "llvm/Transforms/Scalar/LoopRotation.h"
 #include "llvm/Transforms/Scalar/LoopUnrollPass.h"
@@ -77,11 +81,17 @@ static cl::opt<InlinePolicy> JeandleInlinePolicy(
                           "Inline accessor methods only"),
                clEnumValN(InlinePolicy::Off, "off", "Disable inlining")));
 
-Pipeline::Pipeline(OptimizationLevel Level, LLVMContext &Ctx, PipelineMode Mode)
+Pipeline::Pipeline(OptimizationLevel Level, LLVMContext &Ctx, PipelineMode Mode,
+                   TargetMachine *TM)
     : SI(Ctx, /*DebugLogging=*/false) {
   SI.registerCallbacks(PIC, &MAM);
 
-  PassBuilder PB(nullptr, PipelineTuningOptions(), std::nullopt, &PIC);
+  PipelineTuningOptions PTO;
+  // Jeandle uses LLVM's native SLP vectorizer for straight-line Java
+  // stores exposed by pre-PEA unrolling. Keep the pass implementation native;
+  // only opt in through the standard PassBuilder tuning option.
+  PTO.SLPVectorization = true;
+  PassBuilder PB(TM, PTO, std::nullopt, &PIC);
 
   // Register all the basic analyses with the managers.
   PB.registerModuleAnalyses(MAM);
@@ -101,25 +111,26 @@ static void addCoverageVerifier(ModulePassManager &PM) {
     PM.addPass(createModuleToFunctionPassAdaptor(SafepointCoverageVerifier()));
 }
 
-// Prepare strip-mining candidates by exposing array-length exits and scalar
-// comparisons, then hoisting guaranteed invariant header work before rotation
-// applies its duplication budget. The second, speculative LICM cleans up the
-// rotated loop. FunctionToLoopPassAdaptor establishes LoopSimplify and LCSSA
-// form before running the loop pipeline.
-static void addPreparationForStripMining(ModulePassManager &PM) {
+// Canonicalize loops and move invariant guards out of their fast paths before
+// safepoint elimination. Unswitching invalidates and requeues specialized
+// loops, allowing LoopRotate to run again until the primary counted exit is on
+// the latch. The final loop cleanup exposes that shape to strip mining.
+static void addEarlyLoopInvariantAndCanonicalizationOpts(ModulePassManager &PM) {
   PM.addPass(createModuleToFunctionPassAdaptor(EarlyCSEPass()));
   PM.addPass(createModuleToFunctionPassAdaptor(InstCombinePass()));
   PM.addPass(createModuleToFunctionPassAdaptor(SimplifyCFGPass()));
   LoopPassManager LPM;
-  LICMOptions PreRotateLICMOptions;
-  PreRotateLICMOptions.AllowSpeculation = false;
-  LPM.addPass(LICMPass(PreRotateLICMOptions));
   LPM.addPass(LoopRotatePass(true, false));
+
+  LPM.addPass(LICMPass(LICMOptions()));
+  LPM.addPass(SimpleLoopUnswitchPass(/*NonTrivial=*/true));
   LPM.addPass(LICMPass(LICMOptions()));
+  LPM.addPass(SimpleLoopUnswitchPass(/*NonTrivial=*/true));
+
   // IndVarSimplify canonicalizes the IV and strengthens SCEV no-wrap flags (via
   // SimplifyIndVar's getStrengthenedNoWrapFlagsFromBinOp), so the strip-mining
   // no-wrap proofs can rely on SCEV flags instead of hand-derived bounds.
-  LPM.addPass(IndVarSimplifyPass());
+  LPM.addPass(IndVarSimplifyPass(/*WidenIndVars=*/false));
   PM.addPass(createModuleToFunctionPassAdaptor(
       createFunctionToLoopPassAdaptor(std::move(LPM), true)));
 }
@@ -141,10 +152,31 @@ static void addStripMiningPasses(ModulePassManager &PM,
   addCoverageVerifier(PM);
 }
 
+static void addRangeCheckOptPipeline(ModulePassManager &PM) {
+  FunctionPassManager FPM;
+  // Prepare exact Java range-check branches without relying on the ordering
+  // of LLVM's default O3 pipeline. The frontend deliberately emits no
+  // widenable conditions or llvm.experimental.guard intrinsics, so
+  // LoopPredication, GuardWidening, and LowerWidenableCondition would be
+  // no-ops here. IRCE directly recognizes the ordinary conditional branches.
+  FPM.addPass(InstCombinePass());
+  FPM.addPass(SimplifyCFGPass());
+  FPM.addPass(ConstraintEliminationPass());
+
+  FPM.addPass(IRCEPass());
+
+  FPM.addPass(InstCombinePass());
+  FPM.addPass(SimplifyCFGPass());
+
+  PM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+}
+
 ModulePassManager Pipeline::buildJeandlePipeline(PassBuilder &PB,
                                                  OptimizationLevel Level,
                                                  PipelineMode Mode) {
   ModulePassManager PM;
+  const bool StripMiningEnabled = isStripMiningEnabled();
+  const bool DeferEmptyLoopDeletion = (Level == OptimizationLevel::O3);
   PM.addPass(JavaOperationLower(0));
   FunctionPassManager PreCHACleanup;
   // RecoverTypeInfo runs first so that oop loads (including array element
@@ -181,6 +213,30 @@ ModulePassManager Pipeline::buildJeandlePipeline(PassBuilder &PB,
       break;
     }
   }
+
+  PM.addPass(createModuleToFunctionPassAdaptor(JavaOpLengthFolding()));
+  PM.addPass(JavaOperationLower(1));
+  addEarlyLoopInvariantAndCanonicalizationOpts(PM);
+
+  // With strip mining enabled, Early handles only non-loop blocks. Loop polls
+  // remain available to strip mining, then AfterStripMining performs the full
+  // loop-tree deletion. Without strip mining, Early also performs that loop
+  // deletion directly.
+  PM.addPass(createModuleToFunctionPassAdaptor(SafepointPollElimination(
+      SafepointPollEliminationMode::Early, DeferEmptyLoopDeletion)));
+  addCoverageVerifier(PM);
+
+  if (StripMiningEnabled)
+    addStripMiningPasses(PM, DeferEmptyLoopDeletion);
+
+  if (DeferEmptyLoopDeletion) {
+    // Re-form LCSSA independently of strip mining, then atomically delete
+    // finite empty loops and the polls that prevent their deletion.
+    PM.addPass(createModuleToFunctionPassAdaptor(LCSSAPass()));
+    PM.addPass(createModuleToFunctionPassAdaptor(SafepointPollElimination(
+        SafepointPollEliminationMode::LoopDeletionPrep)));
+  }
+
   // ==== PEA segment ====
   // Everything below up to InsertGCBarriers exists to serve PEA: the
   // high-tier loop-optimization cluster that exposes virtualization
@@ -201,7 +257,6 @@ ModulePassManager Pipeline::buildJeandlePipeline(PassBuilder &PB,
     // after PEA — it bloats the analysed loop body without giving PEA any
     // new precision).
     FunctionPassManager PrePEAHighTier;
-    PrePEAHighTier.addPass(JavaOpLengthFolding());
     // Re-establish the canonical loop form; the inline driver may have
     // introduced loops that are not in simplified form.
     PrePEAHighTier.addPass(LoopSimplifyPass());
@@ -211,7 +266,7 @@ ModulePassManager Pipeline::buildJeandlePipeline(PassBuilder &PB,
       // compute trip counts for, deletion removes side-effect-free loops.
       LoopPassManager CanonicalizeLPM;
       CanonicalizeLPM.addPass(LoopRotatePass());
-      CanonicalizeLPM.addPass(IndVarSimplifyPass());
+      CanonicalizeLPM.addPass(IndVarSimplifyPass(/*WidenIndVars=*/false));
       CanonicalizeLPM.addPass(LoopDeletionPass());
       PrePEAHighTier.addPass(
           createFunctionToLoopPassAdaptor(std::move(CanonicalizeLPM)));
@@ -267,11 +322,14 @@ ModulePassManager Pipeline::buildJeandlePipeline(PassBuilder &PB,
     //   fold — allocations, monitors, and helpers whose expanded body would
     //   expose a raw header/klass load that kills virtualization
     //   (load_klass / arraylength / array_store_check / check_if_value_based)
-    //   — carry `"lower-phase"="1"` and are left untouched by
-    //   JavaOperationLower(0) (phase-0 only) and by every pass downstream of
-    //   PEA, surviving until JavaOperationLower(1) below. addrspace(1)
-    //   survives until RewriteStatepointsForGC rewrites it to gc-managed
-    //   pointers.
+    //   — use a later lower phase and are left untouched by
+    //   JavaOperationLower(0). TypeCheckElimination removes statically proven
+    //   array-store checks before PEA; PEA may still fold a residual check
+    //   using its more precise virtual-object state. Any remaining phase-2
+    //   check is reconsidered by post-PEA TypeCheckElimination before
+    //   JavaOperationLower(2). This keeps raw klass loads out of PEA's input.
+    //   addrspace(1) survives until RewriteStatepointsForGC rewrites it to
+    //   gc-managed pointers.
     //
     //   Considered and rejected: a second `PartialEscapeIterative` after the
     //   O2 pipeline. The named intrinsics and addrspace(1) survive through
@@ -289,6 +347,12 @@ ModulePassManager Pipeline::buildJeandlePipeline(PassBuilder &PB,
     //       rounds.
     PM.addPass(createModuleToFunctionPassAdaptor(PartialEscapeIterative()));
   }
+
+  // Keep exact range-check elimination after safepoint strip mining. IRCE can
+  // reshape loops, so run it only after safepoint analysis has consumed the
+  // canonical counted-loop form.
+  addRangeCheckOptPipeline(PM);
+
   // Post-inline type recovery + TCE — unconditional. Runs for both PEA-on
   // (cleans up PEA's materializations) and PEA-off (the default config) so
   // RecoverTypeInfo re-attaches !java-klass metadata stripped by the inline
@@ -301,50 +365,8 @@ ModulePassManager Pipeline::buildJeandlePipeline(PassBuilder &PB,
   PM.addPass(createModuleToFunctionPassAdaptor(ArrayCopySpecialization()));
   PM.addPass(createModuleToFunctionPassAdaptor(TypeCheckElimination()));
 
-  const bool StripMiningEnabled = isStripMiningEnabled();
-  const bool DeferEmptyLoopDeletion = (Level == OptimizationLevel::O3);
-
-  // The loop adaptor establishes LoopSimplify + LCSSA form before
-  // IndVarSimplify or the strip-mining canonicalization pipeline. On the
-  // strip-mining-OFF path, IndVarSimplify also strengthens SCEV no-wrap flags
-  // on the frontend's bare (flagless) IV increments, so Early can prove that a
-  // loop's maximum backedge count is strictly below INT_MAX
-  // (IsIntCountedEquivalent) and drop all of its polls.
-  if (StripMiningEnabled) {
-    addPreparationForStripMining(PM);
-  } else {
-    LoopPassManager LPM;
-    LPM.addPass(IndVarSimplifyPass());
-    PM.addPass(createModuleToFunctionPassAdaptor(
-        createFunctionToLoopPassAdaptor(std::move(LPM))));
-  }
-
-  // With strip mining enabled, Early handles only non-loop blocks. Loop polls
-  // remain available to strip mining, then AfterStripMining performs the full
-  // loop-tree deletion. Without strip mining, Early also performs that loop
-  // deletion directly.
-  PM.addPass(createModuleToFunctionPassAdaptor(SafepointPollElimination(
-      SafepointPollEliminationMode::Early, DeferEmptyLoopDeletion)));
-  addCoverageVerifier(PM);
-
-  if (StripMiningEnabled)
-    addStripMiningPasses(PM, DeferEmptyLoopDeletion);
-
-  // TODO: InsertGCBarriers currently inserts high-level barrier calls before
-  // O3 because it cannot handle O3 generated memory intrinsics and vector
-  // instructions. But the uninlined barrier calls can still block useful
-  // optimizations.
   PM.addPass(createModuleToFunctionPassAdaptor(InsertGCBarriers()));
-
-  if (DeferEmptyLoopDeletion) {
-    // Re-form LCSSA independently of strip mining, then atomically delete
-    // finite empty loops and the polls that prevent their deletion.
-    PM.addPass(createModuleToFunctionPassAdaptor(LCSSAPass()));
-    PM.addPass(createModuleToFunctionPassAdaptor(SafepointPollElimination(
-        SafepointPollEliminationMode::LoopDeletionPrep)));
-  }
-
-  PM.addPass(JavaOperationLower(1));
+  PM.addPass(JavaOperationLower(2));
   PM.addPass(std::move(PB.buildPerModuleDefaultPipeline(Level)));
   PM.addPass(ExpandNarrowOopCast());
   PM.addPass(RewriteStatepointsForGC());

llvm/lib/Transforms/Jeandle/SafepointCoverageVerifier.cpp

@@ -9,8 +9,8 @@
 //===----------------------------------------------------------------------===//
 ///
 /// Verifies that every natural loop either reaches a safepoint on every
-/// backedge path or has a SCEV-provable finite bound accepted by the poll
-/// elimination policy. Strip-mined inner loops are accepted via the
+/// backedge path or has a reproducible finite bound accepted by the poll
+/// elimination policy.  Strip-mined inner loops are accepted via the
 /// "jeandle.strip-mined-poll" attribute on the poll relocated onto the outer
 /// back-edge: this verifier is adjacent to SafepointStripMining in the
 /// pipeline, so the marker is always fresh and trusted.
@@ -26,6 +26,7 @@
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/IR/CFG.h"
+#include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Jeandle/JeandleUtils.h"
@@ -89,7 +90,8 @@ static cl::opt<SafepointCoverageCheck> CoverageCheck(
 #endif
     cl::desc("Safepoint coverage verifier mode."));
 
-static bool isLoopCovered(Loop &L, ScalarEvolution &SE) {
+static bool isLoopCovered(Loop &L, LoopInfo &LI, DominatorTree &DT,
+                          ScalarEvolution &SE) {
   SmallVector<BasicBlock *, 4> Latches;
   L.getLoopLatches(Latches);
   if (Latches.empty())
@@ -104,7 +106,8 @@ static bool isLoopCovered(Loop &L, ScalarEvolution &SE) {
   if (AllLatchesCovered)
     return true;
 
-  jeandle::LoopSafepointFacts Facts = jeandle::LoopSafepointFacts::get(L, SE);
+  jeandle::LoopSafepointFacts Facts =
+      jeandle::LoopSafepointFacts::get(L, LI, DT, SE);
   if (!jeandle::isMarkedStripMinedInner(L) &&
       !jeandle::isStripMiningEnabled() && Facts.IsIntCountedEquivalent)
     return true;
@@ -140,11 +143,12 @@ PreservedAnalyses SafepointCoverageVerifier::run(Function &F,
     return PreservedAnalyses::all();
   }
 
+  auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
   auto &SE = AM.getResult<ScalarEvolutionAnalysis>(F);
 
   bool Broken = false;
   for (Loop *L : LI.getLoopsInPreorder()) {
-    if (isLoopCovered(*L, SE)) {
+    if (isLoopCovered(*L, LI, DT, SE)) {
       LLVM_DEBUG(dbgs() << "  covered: loop " << L->getHeader()->getName()
                         << " in " << F.getName() << "\n");
       continue;