Skip to content

Commit e2ea916

Browse files
committed
iperm hybrid
1 parent 3ea565e commit e2ea916

3 files changed

Lines changed: 80 additions & 38 deletions

File tree

‎src/maxdiffusion/kernels/internal_ring_attention.py‎

Lines changed: 62 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ def _internal_ring_kernel(
109109
use_fixed_m: bool,
110110
uniform_fixed_m: bool,
111111
fixed_m_recenter: float,
112+
precomputed_m: bool,
112113
q_sub: int,
113114
axis_names: tuple[str, ...],
114115
ring_axis: str,
@@ -268,40 +269,69 @@ def _reload_own():
268269
else:
269270
is_fixed = None
270271

271-
def _write_fixed_m():
272-
qf = q_ref[...].astype(float32)
272+
n_q = q_ref.shape[0] // q_sub
273+
two_body_last = HYBRID_TWO_BODY_LAST or (n_q > 1)
274+
275+
def _for_each_q_chunk(fn):
276+
if n_q > 1:
277+
def _qbody(t, carry):
278+
fn(pl.multiple_of(t * q_sub, 128))
279+
return carry
280+
lax.fori_loop(0, n_q, _qbody, None)
281+
else:
282+
for qs in range(0, q_ref.shape[0], q_sub):
283+
fn(qs)
284+
285+
def _write_fixed_m_chunk(qs):
286+
qsl = pl.ds(qs, q_sub)
287+
qf = q_ref[qsl, :].astype(float32)
273288
qn = jnp.sqrt((qf * qf).sum(axis=1))[None, :]
274289
# C(N) comes from the caller (`custom_splash.get_fixed_m_constants`) rather
275290
# than the legacy pinned 88.0: N here is the WHOLE ring's KV length, since
276291
# one launch accumulates every shard into the same VMEM accumulator.
277292
m_fixed = jnp.ceil(qn * mk_ref[0, h]) - fixed_m_recenter
278-
m_scratch_ref[...] = jnp.broadcast_to(m_fixed, m_scratch_ref.shape)
293+
m_scratch_ref[:, qsl] = jnp.broadcast_to(m_fixed, (m_scratch_ref.shape[0], q_sub))
294+
295+
def _init_fixed_chunk(qs):
296+
qsl = pl.ds(qs, q_sub)
297+
o_scratch_ref[:, qsl] = jnp.zeros((o_scratch_ref.shape[0], q_sub), o_scratch_ref.dtype)
298+
l_scratch_ref[:, qsl] = jnp.zeros((l_scratch_ref.shape[0], q_sub), l_scratch_ref.dtype)
299+
if precomputed_m:
300+
if not (fixed_only or two_body_last):
301+
m_scratch_ref[:, qsl] = jnp.full((m_scratch_ref.shape[0], q_sub), mk_ref[0, h], m_scratch_ref.dtype)
302+
else:
303+
_write_fixed_m_chunk(qs)
304+
305+
def _init_online_chunk(qs):
306+
qsl = pl.ds(qs, q_sub)
307+
o_scratch_ref[:, qsl] = jnp.zeros((o_scratch_ref.shape[0], q_sub), o_scratch_ref.dtype)
308+
l_scratch_ref[:, qsl] = jnp.zeros((l_scratch_ref.shape[0], q_sub), l_scratch_ref.dtype)
309+
m_scratch_ref[:, qsl] = jnp.full((m_scratch_ref.shape[0], q_sub), mask_value, m_scratch_ref.dtype)
279310

280311
# ------------------------------------------------------------- accumulate --
281312
@pl.when(is_first_hop)
282313
def _init():
283-
o_scratch_ref[...] = jnp.zeros_like(o_scratch_ref)
284-
l_scratch_ref[...] = jnp.zeros_like(l_scratch_ref)
285314
# Cauchy-Schwarz bound m_i = ceil(||q_i|| * max_j||k_j||) - C, pinned for the
286315
# WHOLE ring. Unlike the external ring this needs no per-hop gating and no
287316
# LSE merge: one kernel sees every shard and the accumulator never leaves
288317
# VMEM, so every hop is already subtracting the identical m. `mk_ref[0, h]`
289-
# is max||k|| reduced over the ring AND ulysses axes by the caller, so the
318+
# is max||k|| reduced over the ring AND ulysses axes by the caller (or, when
319+
# `precomputed_m=True`, the precomputed per-head shift m_h itself), so the
290320
# bound covers keys this rank never holds.
291321
if fixed_only:
292-
_write_fixed_m()
322+
_for_each_q_chunk(_init_fixed_chunk)
293323
elif use_fixed_m:
294324

295325
@pl.when(is_fixed)
296326
def _init_fixed():
297-
_write_fixed_m()
327+
_for_each_q_chunk(_init_fixed_chunk)
298328

299329
@pl.when(jnp.logical_not(is_fixed))
300330
def _init_online():
301-
m_scratch_ref[...] = jnp.full_like(m_scratch_ref, mask_value)
331+
_for_each_q_chunk(_init_online_chunk)
302332

303333
else:
304-
m_scratch_ref[...] = jnp.full_like(m_scratch_ref, mask_value)
334+
_for_each_q_chunk(_init_online_chunk)
305335

306336
def _online_inner(qk, v_chunk, m_prev, l_prev, o_prev):
307337
step = bkv_compute_in
@@ -327,7 +357,7 @@ def _fixed_inner(qk, v_chunk, m_fix, l_prev, o_prev):
327357
# m is constant: no reduce-max over the block and no alpha rescale of o.
328358
step = bkv_compute_in
329359
for c in range(0, qk.shape[0], step):
330-
s_curr = exp(qk[c : c + step] - m_fix[0:1])
360+
s_curr = exp(qk[c : c + step] - m_fix)
331361
l_prev = l_prev + s_curr.sum(axis=0, keepdims=True)
332362
o_prev = o_prev + lax.dot_general(
333363
v_chunk[c : c + step],
@@ -354,7 +384,8 @@ def _one_q_chunk(qs):
354384
qsl = pl.ds(qs, q_sub)
355385
qk = lax.dot_general(k_blk, q_ref[qsl, :], NT_DIM_NUMBERS, preferred_element_type=float32)
356386
if fixed:
357-
l_prev, o_prev = _fixed_inner(qk, v_chunk, m_scratch_ref[:, qsl], l_scratch_ref[:, qsl], o_scratch_ref[:, qsl])
387+
m_fix = mk_ref[0, h] if precomputed_m else m_scratch_ref[:, qsl][0:1]
388+
l_prev, o_prev = _fixed_inner(qk, v_chunk, m_fix, l_scratch_ref[:, qsl], o_scratch_ref[:, qsl])
358389
l_scratch_ref[:, qsl] = l_prev
359390
else:
360391
m_prev, l_prev, o_prev = _online_inner(
@@ -369,15 +400,7 @@ def _one_q_chunk(qs):
369400
# step stalls on instruction fetch -- measured 2.1x slower on pure compute
370401
# (91.2 -> 43.4 ms/layer), invisible in the static schedule, whose bundle
371402
# count and opcode mix stay exactly proportional to the work.
372-
n_q = q_ref.shape[0] // q_sub
373-
if n_q > 1:
374-
def _qbody(t, carry):
375-
_one_q_chunk(pl.multiple_of(t * q_sub, 128))
376-
return carry
377-
lax.fori_loop(0, n_q, _qbody, None)
378-
else:
379-
for qs in range(0, q_ref.shape[0], q_sub):
380-
_one_q_chunk(qs)
403+
_for_each_q_chunk(_one_q_chunk)
381404

382405
def compute_body_fixed(kv_compute_index, _):
383406
_step(kv_compute_index * bkv_compute, bkv_compute, True)
@@ -444,7 +467,6 @@ def _body_online():
444467
# head -- one body. Exact, since online continues from a fixed-m state:
445468
# m_next = max(m_prev, m_curr) and (o, l) rescale by exp2(m_prev - m_next).
446469
# Costs ~0.95 ms/layer vs two bodies, but avoids the ~3x cliff.
447-
two_body_last = HYBRID_TWO_BODY_LAST or (q_ref.shape[0] // q_sub > 1)
448470
if two_body_last:
449471

450472
@pl.when((j == grid_width - 1) & is_fixed)
@@ -467,9 +489,13 @@ def _last_body():
467489
# ordered after this hop's compute.
468490
@pl.when(is_last_hop)
469491
def _write_out():
470-
l = l_scratch_ref[...]
471-
l_inv = jnp.tile(1.0 / l, (head_dim_v_repeats, 1))
472-
o_ref[...] = (o_scratch_ref[...] * l_inv).astype(o_ref.dtype)
492+
def _write_out_chunk(qs):
493+
qsl = pl.ds(qs, q_sub)
494+
l = l_scratch_ref[:, qsl]
495+
l_inv = jnp.tile(1.0 / l, (head_dim_v_repeats, 1))
496+
o_ref[:, qsl] = (o_scratch_ref[:, qsl] * l_inv).astype(o_ref.dtype)
497+
498+
_for_each_q_chunk(_write_out_chunk)
473499

474500

475501
# Whether to single-buffer the q/out blocks when the q block spans the whole head.
@@ -486,6 +512,10 @@ def _write_out():
486512
# instead of online for every head. Two bodies in the last block used to trigger the
487513
# code-size cliff; with the q loop rolled the program may now fit.
488514
HYBRID_TWO_BODY_LAST = False
515+
# Whether `mk[0, h]` carries the precomputed per-head fixed-m shift
516+
# `m_h = ceil(max_i ||q_i|| * max_j ||k_j||) - C` (Fix 2) instead of `max_j ||k_j||`.
517+
import os as _os
518+
PRECOMPUTE_FIXED_M = _os.environ.get("IPERM_PRECOMPUTE_FIXED_M", "1") == "1"
489519

490520
def internal_ring_attention_forward(
491521
q: jax.Array,
@@ -503,6 +533,7 @@ def internal_ring_attention_forward(
503533
uniform_fixed_m: bool = False,
504534
mk: jax.Array | None = None,
505535
fixed_m_recenter: float | None = None,
536+
precomputed_m: bool | None = None,
506537
q_sub_block: int | None = None,
507538
use_experimental_scheduler: bool = False,
508539
vmem_limit_bytes: int | None = None,
@@ -579,6 +610,9 @@ def out_index_map(b, h, i, j, r, *_):
579610
pltpu.SemaphoreType.REGULAR,
580611
]
581612

613+
if precomputed_m is None:
614+
precomputed_m = PRECOMPUTE_FIXED_M
615+
582616
out = pl.pallas_call(
583617
functools.partial(
584618
_internal_ring_kernel,
@@ -596,6 +630,7 @@ def out_index_map(b, h, i, j, r, *_):
596630
use_fixed_m=use_fixed_m,
597631
uniform_fixed_m=uniform_fixed_m,
598632
fixed_m_recenter=fixed_m_recenter,
633+
precomputed_m=bool(precomputed_m),
599634
q_sub=q_sub,
600635
axis_names=tuple(axis_names),
601636
ring_axis=ring_axis,
@@ -640,6 +675,7 @@ def make_internal_ring_attention(
640675
use_fixed_m: bool = False,
641676
uniform_fixed_m: bool = False,
642677
fixed_m_recenter: float | None = None,
678+
precomputed_m: bool | None = None,
643679
q_sub_block: int | None = None,
644680
):
645681
"""Batched `(b, h, s, d) -> (b, h, s, d)` callable. Deliberately NOT vmapped:
@@ -671,6 +707,7 @@ def _ring(q, k, v, mk=None):
671707
uniform_fixed_m=uniform_fixed_m,
672708
mk=mk,
673709
fixed_m_recenter=recenter,
710+
precomputed_m=precomputed_m,
674711
q_sub_block=q_sub_block,
675712
use_experimental_scheduler=use_experimental_scheduler,
676713
vmem_limit_bytes=vmem_limit_bytes,

‎src/maxdiffusion/models/attention_flax.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2414,7 +2414,12 @@ def _run_hybrid(q, k, v, m, km):
24142414
qn_head = jnp.sqrt(qn_head_sq)
24152415
mk_head = jnp.sqrt(mk_head_sq)
24162416
iperm_fixed = jnp.all(qn_head * mk_head <= iperm_bound) & v_ok
2417-
iperm_mk = jnp.stack([mk_head, jnp.ones_like(mk_head)])
2417+
mk_slot0 = (
2418+
jnp.ceil(qn_head * mk_head) - iperm_recenter
2419+
if internal_ring_kernel_mod.PRECOMPUTE_FIXED_M
2420+
else mk_head
2421+
)
2422+
iperm_mk = jnp.stack([mk_slot0, jnp.ones_like(mk_head)])
24182423

24192424
# Whole-head q block: the ring streams each K/V block across the ICI ONCE
24202425
# per head, and the kernel loops over q_sub-sized sub-chunks against it.
@@ -2489,6 +2494,7 @@ def _mk_internal(fixed, recenter=None, uniform=True):
24892494
use_fixed_m=fixed,
24902495
uniform_fixed_m=uniform,
24912496
fixed_m_recenter=recenter,
2497+
precomputed_m=internal_ring_kernel_mod.PRECOMPUTE_FIXED_M,
24922498
)
24932499

24942500
if not use_fixed_m:
@@ -2504,7 +2510,7 @@ def _mk_internal(fixed, recenter=None, uniform=True):
25042510
# `m` is still per query ROW (computed in-kernel from ||q_i||), which
25052511
# is tighter than the external ring's per-Q-block m_B.
25062512
fixed_ok_head = (qn_head * mk_head <= iperm_bound).astype(jnp.float32) * v_ok.astype(jnp.float32)
2507-
mk_hybrid = jnp.stack([mk_head, fixed_ok_head])
2513+
mk_hybrid = jnp.stack([mk_slot0, fixed_ok_head])
25082514
attention_output = _mk_internal(True, iperm_recenter, uniform=False)(query, key, value, mk_hybrid)
25092515
elif fixed_m_uncond:
25102516
# MEASUREMENT VARIANT. Skips the lax.cond and always takes the fixed

‎src/maxdiffusion/utils/tile_size_grid_search.py‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -377,9 +377,9 @@ def bkv_candidates(seq_len: int, *, k: int = 3, align: int = VPU_LANE, max_block
377377
# 98-98.9%, all slow (e.g. R=28160, q_sub=7040, bkv=1280: 233.9 ms vs 146.6 ms
378378
# at bkv=1024); 0.97 would exclude the measured 10 s winner (97.5%).
379379
IPERM_VMEM_MARGIN = 0.98
380-
_IPERM_ROW_BYTES = 1024
381-
_IPERM_SCORE_BYTES = 4.02
382-
_IPERM_KV_BYTES = 1024
380+
_IPERM_ROW_BYTES = 992
381+
_IPERM_SCORE_BYTES = 4.12
382+
_IPERM_KV_BYTES = 512
383383
# Epilogue (_write_out) register-allocator peak: `(o_scratch_ref[...] * l_inv)`
384384
# materializes a full (128, R) f32 intermediate (512*R B) alongside q_ref
385385
# (256*R B), o_ref (256*R B), o_scratch_ref (512*R B), m/l_scratch_ref (64*R B),
@@ -390,11 +390,10 @@ def bkv_candidates(seq_len: int, *, k: int = 3, align: int = VPU_LANE, max_block
390390
_IPERM_EPILOGUE_KV_BYTES = 1536
391391
# block_q_outer < R switches the kernel from the single-buffered resident Q to
392392
# double-buffered Q/output blocks (RESIDENT_SINGLE_BUFFER gate in
393-
# internal_ring_attention.py). No sweep has exercised it yet, so this is an
394-
# UNCALIBRATED upper estimate (1.5x the per-row cost plus 3 MiB of slack); the
395-
# run-time OOM classification is the backstop.
396-
_IPERM_OUTER_ROW_BYTES = 1536
397-
_IPERM_OUTER_SLACK_BYTES = 3 * 1024 * 1024
393+
# internal_ring_attention.py). Calibrated against DP2-CP4 12.5 s outer=23040
394+
# sweeps (bkv=1280..3072, q_sub=4608..7680).
395+
_IPERM_OUTER_ROW_BYTES = 1472
396+
_IPERM_OUTER_SLACK_BYTES = 1 * 1024 * 1024
398397

399398
# Planner knobs (see `iperm_candidates`).
400399
IPERM_MAX_CANDIDATES = 30
@@ -404,9 +403,9 @@ def bkv_candidates(seq_len: int, *, k: int = 3, align: int = VPU_LANE, max_block
404403
IPERM_RESIDENT_TOP_K = 3 # best-scoring resident lengths kept from that sweep
405404
IPERM_MAX_OUTER_BLOCKS = 8 # block_q_outer: at most 8 ring re-walks
406405
IPERM_BKV_ANCHORS = (1024, 1280)
407-
IPERM_BKV_EXTRAS = (512, 768, 1536, 2048)
406+
IPERM_BKV_EXTRAS = (512, 768, 1536, 1664, 1792, 2048, 2560)
408407
# block_q_sub values that won (or tied for best) across the measured sweeps.
409-
IPERM_KNOWN_GOOD_Q_SUB = (7040, 6272, 5760, 4736, 3840, 3584, 2816, 2688, 1920, 1664)
408+
IPERM_KNOWN_GOOD_Q_SUB = (7040, 6272, 5760, 4736, 3840, 3712, 3584, 3328, 3072, 2816, 2688, 2560, 1920, 1664)
410409
# block_q_sub admissibility (see `_q_sub_admissible`). Every candidate walks R
411410
# in at least two chunks: a single whole-block chunk is the instruction-memory
412411
# cliff (the static program grows with R; ~2.1x slower). Formula-driven
@@ -769,7 +768,7 @@ def iperm_outer_candidates(
769768
budget=margin * vmem_bytes,
770769
max_candidates=max_candidates,
771770
ladder_bqs=[bq for bq in ladder_bqs if bq >= VPU_LANE and bq % VPU_LANE == 0],
772-
bkvs=(*anchor_bkvs, min(512, kv_cap)),
771+
bkvs=(*anchor_bkvs, min(1536, kv_cap), min(512, kv_cap)),
773772
)
774773

775774

0 commit comments

Comments
 (0)