Focus call-graph macrobenchmarks on analyses

core/src/jmh/java/com/ibm/wala/benchmarks/callgraph/CallGraphBenchmark.java

@@ -1,25 +1,49 @@
 package com.ibm.wala.benchmarks.callgraph;
 
 import com.ibm.wala.core.tests.callGraph.CallGraphTest;
+import com.ibm.wala.core.tests.callGraph.CallGraphTestUtil;
+import com.ibm.wala.core.tests.util.TestConstants;
+import com.ibm.wala.ipa.callgraph.AnalysisCacheImpl;
+import com.ibm.wala.ipa.callgraph.AnalysisOptions;
+import com.ibm.wala.ipa.callgraph.AnalysisScope;
+import com.ibm.wala.ipa.callgraph.CallGraph;
+import com.ibm.wala.ipa.callgraph.Entrypoint;
+import com.ibm.wala.ipa.callgraph.impl.AllApplicationEntrypoints;
+import com.ibm.wala.ipa.cha.ClassHierarchy;
 import com.ibm.wala.ipa.cha.ClassHierarchyException;
+import com.ibm.wala.ipa.cha.ClassHierarchyFactory;
 import com.ibm.wala.util.CancelException;
 import java.io.IOException;
 import java.util.concurrent.TimeUnit;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
 import org.openjdk.jmh.annotations.Measurement;
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
 import org.openjdk.jmh.annotations.Warmup;
 
 /**
- * JMH macrobenchmarks that perform large-scale, end-to-end WALA analyses.
+ * JMH macrobenchmarks that perform large-scale, end-to-end WALA call-graph analyses.
  *
  * <p>Unlike the microbenchmarks, which measure tightly-scoped operations in isolation, each
  * macrobenchmark here runs a complete analysis task just as a real WALA user or a unit test would.
- * The first such task, {@link #testHelloAllEntrypoints()}, is exactly the work of {@link
- * CallGraphTest#testHelloAllEntrypoints}.
+ *
+ * <p>This class contains two kinds of benchmarks:
+ *
+ * <ul>
+ *   <li>{@link #testHelloAllEntrypoints()} — a faithful reproduction of {@link
+ *       CallGraphTest#testHelloAllEntrypoints} including all test-harness verification. This is the
+ *       existing end-to-end probe.
+ *   <li>Per-phase analysis-only benchmarks ({@link #buildRTA()}, {@link #buildZeroCFA()}, etc.)
+ *       that time <em>only</em> the pointer-analysis construction for each call-graph algorithm,
+ *       with no verification overhead. The scope, class hierarchy, entrypoints, analysis options,
+ *       and cache are built once in {@link #setup()}, outside the timed region.
+ * </ul>
  *
  * <p>Because these tasks take seconds rather than microseconds, they use the {@link
  * Mode#SingleShotTime single-shot time mode}, in which each iteration is a single invocation. Run
@@ -35,9 +59,26 @@
 @Fork(6)
 @Measurement(iterations = 6)
 @OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Thread)
 @Warmup(iterations = 3)
 public class CallGraphBenchmark {
 
+  private AnalysisScope scope;
+  private ClassHierarchy cha;
+  private AnalysisOptions options;
+  private AnalysisCacheImpl cache;
+
+  @Setup(Level.Trial)
+  public void setup() throws ClassHierarchyException, IllegalArgumentException, IOException {
+    scope =
+        CallGraphTestUtil.makeJ2SEAnalysisScope(
+            TestConstants.HELLO, CallGraphTestUtil.REGRESSION_EXCLUSIONS);
+    cha = ClassHierarchyFactory.make(scope);
+    Iterable<Entrypoint> entrypoints = new AllApplicationEntrypoints(scope, cha);
+    options = CallGraphTestUtil.makeAnalysisOptions(scope, entrypoints);
+    cache = new AnalysisCacheImpl();
+  }
+
   /**
    * Builds call graphs from all application entrypoints of the {@code hello} test subject.
    *
@@ -51,4 +92,55 @@ public void testHelloAllEntrypoints()
       throws CancelException, ClassHierarchyException, IllegalArgumentException, IOException {
     new CallGraphTest().testHelloAllEntrypoints();
   }
+
+  /**
+   * Times only the RTA call-graph construction, with no test-harness verification.
+   *
+   * <p>The preceding pipeline work (scope, CHA, options, cache) is done once in {@link #setup()},
+   * before any timing begins. Returning the built {@link CallGraph} prevents dead-code elimination.
+   */
+  @Benchmark
+  public CallGraph buildRTA() throws IllegalArgumentException, CancelException {
+    return CallGraphTestUtil.buildRTA(options, cache, cha);
+  }
+
+  /** Times only the 0-CFA call-graph construction, with no test-harness verification. */
+  @Benchmark
+  public CallGraph buildZeroCFA() throws IllegalArgumentException, CancelException {
+    return CallGraphTestUtil.buildZeroCFA(options, cache, cha, false);
+  }
+
+  /** Times only the 0-1-CFA call-graph construction, with no test-harness verification. */
+  @Benchmark
+  public CallGraph buildZeroOneCFA() throws IllegalArgumentException, CancelException {
+    return CallGraphTestUtil.buildZeroOneCFA(options, cache, cha, false);
+  }
+
+  /** Times only the 0-Container-CFA call-graph construction, with no test-harness verification. */
+  @Benchmark
+  public CallGraph buildZeroContainerCFA() throws IllegalArgumentException, CancelException {
+    return CallGraphTestUtil.buildZeroContainerCFA(options, cache, cha);
+  }
+
+  /**
+   * Times only the 0-1-Container-CFA call-graph construction, with no test-harness verification.
+   */
+  @Benchmark
+  public CallGraph buildZeroOneContainerCFA() throws IllegalArgumentException, CancelException {
+    return CallGraphTestUtil.buildZeroOneContainerCFA(options, cache, cha);
+  }
+
+  /**
+   * Times all five call-graph phases in sequence, sharing one cache and JIT regime as in the real
+   * pipeline, but with no test-harness verification.
+   *
+   * <p>This is the lighter-weight counterpart to {@link #testHelloAllEntrypoints()}: it preserves
+   * the ensemble behavior (phases share one cache, exactly as a user would run them) while dropping
+   * the ~30% verification overhead. Returning the built call graphs prevents dead-code elimination.
+   */
+  @Benchmark
+  public CallGraphTest.AllApplicationCallGraphs buildAllCallGraphs()
+      throws IllegalArgumentException, CancelException {
+    return CallGraphTest.buildAllCallGraphs(options, cache, cha, false);
+  }
 }

core/src/test/java/com/ibm/wala/core/tests/callGraph/CallGraphTest.java

@@ -434,6 +434,39 @@ public static Iterable<Entrypoint> makePrimordialMainEntrypoints(ClassHierarchy
     return result;
   }
 
+  /**
+   * The five call graphs computed by {@link #doCallGraphs}, in build order: RTA, 0-CFA, 0-1-CFA,
+   * 0-Container-CFA, and 0-1-Container-CFA.
+   */
+  public record AllApplicationCallGraphs(
+      CallGraph rta,
+      CallGraph zeroCFA,
+      CallGraph zeroOneCFA,
+      CallGraph zeroContainerCFA,
+      CallGraph zeroOneContainerCFA) {}
+
+  /**
+   * Builds the five call graphs that {@link #doCallGraphs} computes, sharing {@code cache} across
+   * all phases exactly as the real pipeline does.
+   *
+   * <p>No verification (graph-integrity checks, squashing, or stat printing) is performed here, so
+   * benchmarks can time precisely this analysis without test-harness overhead.
+   */
+  public static AllApplicationCallGraphs buildAllCallGraphs(
+      AnalysisOptions options,
+      IAnalysisCacheView cache,
+      IClassHierarchy cha,
+      boolean testPAToString)
+      throws IllegalArgumentException, CancelException {
+    CallGraph rta = CallGraphTestUtil.buildRTA(options, cache, cha);
+    CallGraph zeroCFA = CallGraphTestUtil.buildZeroCFA(options, cache, cha, testPAToString);
+    CallGraph zeroOneCFA = CallGraphTestUtil.buildZeroOneCFA(options, cache, cha, testPAToString);
+    CallGraph zeroContainerCFA = CallGraphTestUtil.buildZeroContainerCFA(options, cache, cha);
+    CallGraph zeroOneContainerCFA = CallGraphTestUtil.buildZeroOneContainerCFA(options, cache, cha);
+    return new AllApplicationCallGraphs(
+        rta, zeroCFA, zeroOneCFA, zeroContainerCFA, zeroOneContainerCFA);
+  }
+
   public static void doCallGraphs(
       AnalysisOptions options, IAnalysisCacheView cache, IClassHierarchy cha)
       throws IllegalArgumentException, CancelException {
@@ -447,11 +480,12 @@ public static void doCallGraphs(
       IClassHierarchy cha,
       boolean testPAToString)
       throws IllegalArgumentException, CancelException {
+    AllApplicationCallGraphs graphs = buildAllCallGraphs(options, cache, cha, testPAToString);
 
     // ///////////////
     // // RTA /////
     // ///////////////
-    final CallGraph rta = CallGraphTestUtil.buildRTA(options, cache, cha);
+    final CallGraph rta = graphs.rta();
     assertThatCode(() -> GraphIntegrity.check(rta)).doesNotThrowAnyException();
 
     Set<MethodReference> rtaMethods = CallGraphStats.collectMethods(rta);
@@ -462,12 +496,10 @@ public static void doCallGraphs(
     // ///////////////
     // // 0-CFA /////
     // ///////////////
-    CallGraph cg = CallGraphTestUtil.buildZeroCFA(options, cache, cha, testPAToString);
-
     // FIXME: annoying special cases caused by clone2assign mean using
     // the rta graph for proper graph subset checking does not work.
     // (note that all the other such checks do use proper graph subset)
-    Graph<MethodReference> squashZero = checkCallGraph(cg, null, "0-CFA");
+    Graph<MethodReference> squashZero = checkCallGraph(graphs.zeroCFA(), null, "0-CFA");
 
     // test Pretransitive 0-CFA
     // not currently supported
@@ -483,24 +515,23 @@ public static void doCallGraphs(
     // ///////////////
     // // 0-1-CFA ///
     // ///////////////
-    cg = CallGraphTestUtil.buildZeroOneCFA(options, cache, cha, testPAToString);
-    Graph<MethodReference> squashZeroOne = checkCallGraph(cg, squashZero, "0-1-CFA");
+    Graph<MethodReference> squashZeroOne =
+        checkCallGraph(graphs.zeroOneCFA(), squashZero, "0-1-CFA");
 
     // ///////////////////////////////////////////////////
     // // 0-CFA augmented to disambiguate containers ///
     // ///////////////////////////////////////////////////
-    cg = CallGraphTestUtil.buildZeroContainerCFA(options, cache, cha);
-    Graph<MethodReference> squashZeroContainer = checkCallGraph(cg, squashZero, "0-Container-CFA");
+    Graph<MethodReference> squashZeroContainer =
+        checkCallGraph(graphs.zeroContainerCFA(), squashZero, "0-Container-CFA");
 
     // ///////////////////////////////////////////////////
     // // 0-1-CFA augmented to disambiguate containers ///
     // ///////////////////////////////////////////////////
-    cg = CallGraphTestUtil.buildZeroOneContainerCFA(options, cache, cha);
-    checkCallGraph(cg, squashZeroContainer, "0-1-Container-CFA");
-    checkCallGraph(cg, squashZeroOne, "0-1-Container-CFA");
+    checkCallGraph(graphs.zeroOneContainerCFA(), squashZeroContainer, "0-1-Container-CFA");
+    checkCallGraph(graphs.zeroOneContainerCFA(), squashZeroOne, "0-1-Container-CFA");
 
     // test ICFG
-    checkICFG(cg);
+    checkICFG(graphs.zeroOneContainerCFA());
     // /////////////
     // // 1-CFA ///
     // /////////////