Merge master into fix-flaky-SchedulerManager-close-deadlock

Preserve the upstream bounded scheduler shutdown implementation and retain the membership-lock regression test.

Author: lhotari

.github/actions/setup-gradle/action.yml

@@ -18,7 +18,7 @@
 #
 
 name: Setup Gradle
-description: Sets up Gradle with Develocity or public build scan publishing by default
+description: Sets up Gradle with runner memory defaults and build scans for public repositories
 inputs:
   develocity-access-key:
     description: 'Develocity access key for authenticated build scans'
@@ -28,6 +28,10 @@ inputs:
     description: 'Whether to publish build scans or use Develocity when the access key is set'
     required: false
     default: 'true'
+  memory-profile:
+    description: 'Gradle memory profile: auto (low-memory on Linux runners with up to 8 GiB RAM), low-memory, or standard'
+    required: false
+    default: 'auto'
   cache-read-only:
     description: 'Whether the Gradle cache is read-only'
     required: false
@@ -40,7 +44,7 @@ runs:
   using: composite
   steps:
     - name: Set Develocity Project ID and configure custom settings
-      if: ${{ inputs.develocity-access-key != '' && inputs.build-scan-publish == 'true' }}
+      if: ${{ inputs.develocity-access-key != '' && inputs.build-scan-publish == 'true' && github.event.repository.visibility == 'public' }}
       shell: bash
       run: |
         mkdir -p ~/.gradle
@@ -49,23 +53,29 @@ runs:
         grep -q 'systemProp.scan.uploadInBackground=' ~/.gradle/gradle.properties || echo systemProp.scan.uploadInBackground=false >> ~/.gradle/gradle.properties
 
     - name: Setup Gradle with Develocity
-      if: ${{ inputs.develocity-access-key != '' && inputs.build-scan-publish == 'true' }}
+      if: ${{ inputs.develocity-access-key != '' && inputs.build-scan-publish == 'true' && github.event.repository.visibility == 'public' }}
       uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0
       with:
         develocity-injection-enabled: true
         develocity-url: https://develocity.apache.org
         # expected format is develocity.apache.org:<access-key>
         develocity-access-key: ${{ inputs.develocity-access-key }}
-        build-scan-publish: ${{ inputs.build-scan-publish }}
+        build-scan-publish: ${{ inputs.build-scan-publish == 'true' && github.event.repository.visibility == 'public' }}
         cache-read-only: ${{ inputs.cache-read-only }}
         add-job-summary: ${{ inputs.add-job-summary }}
 
     - name: Setup Gradle
-      if: ${{ !(inputs.develocity-access-key != '' && inputs.build-scan-publish == 'true') }}
+      if: ${{ !(inputs.develocity-access-key != '' && inputs.build-scan-publish == 'true' && github.event.repository.visibility == 'public') }}
       uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0
       with:
-        build-scan-publish: ${{ inputs.build-scan-publish }}
+        build-scan-publish: ${{ inputs.build-scan-publish == 'true' && github.event.repository.visibility == 'public' }}
         build-scan-terms-of-use-url: 'https://gradle.com/terms-of-service'
         build-scan-terms-of-use-agree: 'yes'
         cache-read-only: ${{ inputs.cache-read-only }}
-        add-job-summary: ${{ inputs.add-job-summary }}
+        add-job-summary: ${{ inputs.add-job-summary }}
+
+    - name: Configure Gradle memory
+      shell: bash
+      env:
+        MEMORY_PROFILE: ${{ inputs.memory-profile }}
+      run: bash "$GITHUB_ACTION_PATH/configure-memory.sh" "$MEMORY_PROFILE"

.github/actions/setup-gradle/configure-memory.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -euo pipefail
+
+profile="${1:-auto}"
+case "$profile" in
+  auto)
+    profile=standard
+    if [[ -r /proc/meminfo ]]; then
+      memory_kib=$(awk '/^MemTotal:/ { print $2 }' /proc/meminfo)
+      if [[ "$memory_kib" =~ ^[0-9]+$ ]] && (( memory_kib > 0 && memory_kib <= 8 * 1024 * 1024 )); then
+        profile=low-memory
+      fi
+    fi
+    ;;
+  low-memory|standard) ;;
+  *) echo "Unknown Gradle memory profile: $profile" >&2; exit 1 ;;
+esac
+
+echo "Gradle memory profile: $profile"
+if [[ "$profile" == low-memory ]]; then
+  gradle_dir="${GRADLE_USER_HOME:-$HOME/.gradle}"
+  mkdir -p "$gradle_dir"
+  # Bound workers across projects as well as forks within each test task. Leave
+  # room for native JVM memory and containers, and recycle accumulated test state.
+  printf '\n' >> "$gradle_dir/gradle.properties"
+  cat >> "$gradle_dir/gradle.properties" <<'EOF'
+org.gradle.jvmargs=-Xmx2g -Xss2m -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp
+org.gradle.workers.max=2
+testMaxParallelForks=2
+testForkEvery=50
+EOF
+fi

.github/workflows/ci-go-functions.yaml

@@ -31,8 +31,15 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+# Do not depend on the repository or enterprise default token permissions.
+permissions:
+  contents: read
+
 jobs:
   preconditions:
+    permissions:
+      contents: read
+      pull-requests: read # List changed files and check whether the PR is ready.
     name: Preconditions
     runs-on: ubuntu-24.04
     outputs:

.github/workflows/ci-python-functions.yaml

@@ -33,8 +33,15 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+# Do not depend on the repository or enterprise default token permissions.
+permissions:
+  contents: read
+
 jobs:
   preconditions:
+    permissions:
+      contents: read
+      pull-requests: read # List changed files and check whether the PR is ready.
     name: Preconditions
     runs-on: ubuntu-24.04
     outputs:

.github/workflows/ci-semantic-pull-request.yml

@@ -29,6 +29,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.number }}
   cancel-in-progress: true
 
+permissions:
+  pull-requests: read
+
 jobs:
   main:
     name: Check pull request title

.github/workflows/codeql.yaml

@@ -36,7 +36,7 @@ env:
 jobs:
   analyze:
     # only run on push and schedule in apache/pulsar repo
-    if: ${{ github.repository == 'apache/pulsar' || github.event_name == 'workflow_dispatch' }}
+    if: ${{ (github.repository == 'apache/pulsar' || github.event_name == 'workflow_dispatch') && (github.event.repository.visibility == 'public' || vars.CI_ENABLE_CODEQL == 'true') }}
     name: Analyze
     runs-on: 'ubuntu-latest'
     timeout-minutes: 360

.github/workflows/pulsar-ci-flaky.yaml

@@ -81,8 +81,16 @@ env:
   ARTIFACT_RETENTION_DAYS: 3
   JDK_DISTRIBUTION: corretto
 
+# Do not depend on the repository or enterprise default token permissions.
+permissions:
+  contents: read
+
 jobs:
   preconditions:
+    permissions:
+      contents: read
+      pull-requests: read # List changed files and check whether the PR is ready.
+      actions: write # Cancel scheduled runs in forks.
     name: Preconditions
     runs-on: ubuntu-24.04
     outputs:

.github/workflows/pulsar-ci.yaml

@@ -76,8 +76,16 @@ env:
   ARTIFACT_RETENTION_DAYS: 3
   JDK_DISTRIBUTION: corretto
 
+# Do not depend on the repository or enterprise default token permissions.
+permissions:
+  contents: read
+
 jobs:
   preconditions:
+    permissions:
+      contents: read
+      pull-requests: read # List changed files and check whether the PR is ready.
+      actions: write # Cancel scheduled runs in forks.
     name: Preconditions
     runs-on: ubuntu-24.04
     outputs:
@@ -276,8 +284,8 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
-      - name: Clean Disk when tracing test resource cleanup
-        if: ${{ env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }}
+      - name: Clean Disk for replication tests or resource tracing
+        if: ${{ matrix.group == 'BROKER_GROUP_5' || env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }}
         uses: ./.github/actions/clean-disk
 
       - name: Setup ssh access to build runner VM
@@ -638,6 +646,9 @@ jobs:
           action: wait
 
   pulsar-test-latest-version-image:
+    permissions:
+      contents: read
+      security-events: write # Upload scheduled container scan results.
     name: Build pulsar-test-latest-version Docker image
     runs-on: ubuntu-24.04
     timeout-minutes: 60
@@ -855,7 +866,8 @@ jobs:
     runs-on: ubuntu-24.04
     timeout-minutes: 60
     needs: ['preconditions', 'unit-tests']
-    if: ${{ (needs.preconditions.outputs.java_non_tests == 'true' || github.event_name != 'pull_request') && ((github.event_name == 'pull_request' && github.base_ref == 'master') || (github.event_name != 'pull_request' && github.ref_name == 'master')) }}
+    # Private repositories need Code Security enabled before opting in.
+    if: ${{ (github.event.repository.visibility == 'public' || vars.CI_ENABLE_CODEQL == 'true') && (needs.preconditions.outputs.java_non_tests == 'true' || github.event_name != 'pull_request') && ((github.event_name == 'pull_request' && github.base_ref == 'master') || (github.event_name != 'pull_request' && github.ref_name == 'master')) }}
     permissions:
       actions: read
       contents: read
@@ -916,6 +928,8 @@ jobs:
   # protected_branches section for master branch required_status_checks.
   # It depends on all other jobs in this workflow.
   pulsar-ci-checks-completed:
+    permissions:
+      actions: write # Delete intermediate build artifacts.
     name: "Pulsar CI checks completed"
     # run always, but skip for other repositories than apache/pulsar when a scheduled workflow is cancelled
     # this is to allow the workflow scheduled jobs to show as cancelled instead of failed since scheduled

CODING.md

@@ -87,8 +87,9 @@ Pulsar relies heavily on `CompletableFuture`; prefer it over `ListenableFuture`
   return firstAsync(arg).thenCompose(v -> secondAsync(v));
   ```
 - **Converting a synchronous-throwing method to a failed future is not mechanical** — some callers rely
-  on the throw happening *before* the async work starts, so evaluate each call site. Use a shared
-  `checkArgumentAsync` helper (in `FutureUtil`) to validate without duplicating try/catch.
+  on the throw happening *before* the async work starts, so evaluate each call site. Use shared
+  `FutureUtil` helpers, such as `supplySafely` for invoking a `Supplier<CompletableFuture<T>>`,
+  instead of duplicating try/catch or null-future handling.
 
 - **Limit concurrency and handle backpressure.** Firing many async operations at once can overwhelm the
   system. Options:

CONTRIBUTING.md

@@ -105,6 +105,23 @@ Other test-related properties: `-PtestJavaVersion=17` (run tests on a different
 `-PtestRetryCount=N`, `-PtestFailFast=true|false`, `-PprotobufVersion=4.31.1` (protobuf v4
 compatibility tests).
 
+Test JVMs default to a `1300m` heap and up to four forks per task. Override these with
+`-PtestMaxHeapSize=1500m` and `-PtestMaxParallelForks=2`; `--max-workers=2` also limits
+concurrent workers across projects. Task-specific limits, such as the integration tests' single
+fork and `1G` heap, take precedence.
+
+Use `-PtestForkEvery=50` to replace a test JVM after 50 detected test classes, limiting state
+retained across classes. The default is `0` (no class limit). Gradle counts candidate classes
+passed to test workers before TestNG group filtering, so classes excluded by groups still count
+toward each batch. `--tests` can narrow candidates, but wildcard filters may still count classes
+that execute no matching tests. Test methods, data-provider rows and factory instances do not
+count separately.
+Tasks using TestNG XML suites and async-profiler keep recycling disabled; task-specific
+isolation, such as SASL's one class per worker, takes precedence.
+
+Test JVMs write heap dumps on heap exhaustion to `/tmp/java_pid<PID>.hprof`, collected by CI's
+existing failure artifacts. Set `-PtestHeapDumpPath=<existing-directory>` to use another directory.
+
 Failed tests are retried once by default (`testRetryCount=1`; `0` when running inside the IDE). When
 running tests locally, prefer **`-PtestRetryCount=0`** to catch failures (including flakiness) early
 instead of having retries mask them.
@@ -333,6 +350,24 @@ locally. (`integrationTest` also accepts `-PtestGroups` / `-PexcludedTestGroups`
 
 ### Running the full CI pipeline (Personal CI)
 
+The shared `setup-gradle` action automatically selects a smaller memory profile on Linux runners
+with up to 8 GiB of physical RAM: a `2g` Gradle heap, two workers across projects, up to two test
+forks per task, and new test workers after 50 detected classes. Larger runners retain the usual
+settings. The action's `memory-profile` input accepts `auto` (default), `low-memory` (force the
+smaller profile), or `standard` (leave the memory settings unchanged). Command-line `-Dorg.gradle.jvmargs`,
+`--max-workers` and `-Ptest*` options can override the profile settings.
+
+Develocity injection and build-scan publishing are disabled when the workflow repository is
+private or its visibility is unavailable, even if an access key is configured. Public repositories
+can also disable them with the action's `build-scan-publish: 'false'` input. CodeQL runs by default
+for public repositories; private repositories with GitHub Code Security enabled can opt in by
+setting the repository variable `CI_ENABLE_CODEQL=true`.
+
+The CI workflows declare the token permissions needed by their jobs, including reading PR changes.
+This supports repositories whose organization or enterprise enforces read-only default workflow
+permissions. Explicit permissions do not override restrictions on tokens for fork pull requests or
+enterprise policies that prohibit particular actions.
+
 The full test suite is large and slow to run locally. While iterating on a change, run only the
 narrowly-scoped tests relevant to the change (a single test class or package, see above) rather than
 a module's entire test task. To validate a larger change against the **full** CI pipeline, do not run