@@ -109,6 +109,7 @@ def _internal_ring_kernel(
109109 use_fixed_m : bool ,
110110 uniform_fixed_m : bool ,
111111 fixed_m_recenter : float ,
112+ precomputed_m : bool ,
112113 q_sub : int ,
113114 axis_names : tuple [str , ...],
114115 ring_axis : str ,
@@ -268,40 +269,69 @@ def _reload_own():
268269 else :
269270 is_fixed = None
270271
271- def _write_fixed_m ():
272- qf = q_ref [...].astype (float32 )
272+ n_q = q_ref .shape [0 ] // q_sub
273+ two_body_last = HYBRID_TWO_BODY_LAST or (n_q > 1 )
274+
275+ def _for_each_q_chunk (fn ):
276+ if n_q > 1 :
277+ def _qbody (t , carry ):
278+ fn (pl .multiple_of (t * q_sub , 128 ))
279+ return carry
280+ lax .fori_loop (0 , n_q , _qbody , None )
281+ else :
282+ for qs in range (0 , q_ref .shape [0 ], q_sub ):
283+ fn (qs )
284+
285+ def _write_fixed_m_chunk (qs ):
286+ qsl = pl .ds (qs , q_sub )
287+ qf = q_ref [qsl , :].astype (float32 )
273288 qn = jnp .sqrt ((qf * qf ).sum (axis = 1 ))[None , :]
274289 # C(N) comes from the caller (`custom_splash.get_fixed_m_constants`) rather
275290 # than the legacy pinned 88.0: N here is the WHOLE ring's KV length, since
276291 # one launch accumulates every shard into the same VMEM accumulator.
277292 m_fixed = jnp .ceil (qn * mk_ref [0 , h ]) - fixed_m_recenter
278- m_scratch_ref [...] = jnp .broadcast_to (m_fixed , m_scratch_ref .shape )
293+ m_scratch_ref [:, qsl ] = jnp .broadcast_to (m_fixed , (m_scratch_ref .shape [0 ], q_sub ))
294+
295+ def _init_fixed_chunk (qs ):
296+ qsl = pl .ds (qs , q_sub )
297+ o_scratch_ref [:, qsl ] = jnp .zeros ((o_scratch_ref .shape [0 ], q_sub ), o_scratch_ref .dtype )
298+ l_scratch_ref [:, qsl ] = jnp .zeros ((l_scratch_ref .shape [0 ], q_sub ), l_scratch_ref .dtype )
299+ if precomputed_m :
300+ if not (fixed_only or two_body_last ):
301+ m_scratch_ref [:, qsl ] = jnp .full ((m_scratch_ref .shape [0 ], q_sub ), mk_ref [0 , h ], m_scratch_ref .dtype )
302+ else :
303+ _write_fixed_m_chunk (qs )
304+
305+ def _init_online_chunk (qs ):
306+ qsl = pl .ds (qs , q_sub )
307+ o_scratch_ref [:, qsl ] = jnp .zeros ((o_scratch_ref .shape [0 ], q_sub ), o_scratch_ref .dtype )
308+ l_scratch_ref [:, qsl ] = jnp .zeros ((l_scratch_ref .shape [0 ], q_sub ), l_scratch_ref .dtype )
309+ m_scratch_ref [:, qsl ] = jnp .full ((m_scratch_ref .shape [0 ], q_sub ), mask_value , m_scratch_ref .dtype )
279310
280311 # ------------------------------------------------------------- accumulate --
281312 @pl .when (is_first_hop )
282313 def _init ():
283- o_scratch_ref [...] = jnp .zeros_like (o_scratch_ref )
284- l_scratch_ref [...] = jnp .zeros_like (l_scratch_ref )
285314 # Cauchy-Schwarz bound m_i = ceil(||q_i|| * max_j||k_j||) - C, pinned for the
286315 # WHOLE ring. Unlike the external ring this needs no per-hop gating and no
287316 # LSE merge: one kernel sees every shard and the accumulator never leaves
288317 # VMEM, so every hop is already subtracting the identical m. `mk_ref[0, h]`
289- # is max||k|| reduced over the ring AND ulysses axes by the caller, so the
318+ # is max||k|| reduced over the ring AND ulysses axes by the caller (or, when
319+ # `precomputed_m=True`, the precomputed per-head shift m_h itself), so the
290320 # bound covers keys this rank never holds.
291321 if fixed_only :
292- _write_fixed_m ( )
322+ _for_each_q_chunk ( _init_fixed_chunk )
293323 elif use_fixed_m :
294324
295325 @pl .when (is_fixed )
296326 def _init_fixed ():
297- _write_fixed_m ( )
327+ _for_each_q_chunk ( _init_fixed_chunk )
298328
299329 @pl .when (jnp .logical_not (is_fixed ))
300330 def _init_online ():
301- m_scratch_ref [...] = jnp . full_like ( m_scratch_ref , mask_value )
331+ _for_each_q_chunk ( _init_online_chunk )
302332
303333 else :
304- m_scratch_ref [...] = jnp . full_like ( m_scratch_ref , mask_value )
334+ _for_each_q_chunk ( _init_online_chunk )
305335
306336 def _online_inner (qk , v_chunk , m_prev , l_prev , o_prev ):
307337 step = bkv_compute_in
@@ -327,7 +357,7 @@ def _fixed_inner(qk, v_chunk, m_fix, l_prev, o_prev):
327357 # m is constant: no reduce-max over the block and no alpha rescale of o.
328358 step = bkv_compute_in
329359 for c in range (0 , qk .shape [0 ], step ):
330- s_curr = exp (qk [c : c + step ] - m_fix [ 0 : 1 ] )
360+ s_curr = exp (qk [c : c + step ] - m_fix )
331361 l_prev = l_prev + s_curr .sum (axis = 0 , keepdims = True )
332362 o_prev = o_prev + lax .dot_general (
333363 v_chunk [c : c + step ],
@@ -354,7 +384,8 @@ def _one_q_chunk(qs):
354384 qsl = pl .ds (qs , q_sub )
355385 qk = lax .dot_general (k_blk , q_ref [qsl , :], NT_DIM_NUMBERS , preferred_element_type = float32 )
356386 if fixed :
357- l_prev , o_prev = _fixed_inner (qk , v_chunk , m_scratch_ref [:, qsl ], l_scratch_ref [:, qsl ], o_scratch_ref [:, qsl ])
387+ m_fix = mk_ref [0 , h ] if precomputed_m else m_scratch_ref [:, qsl ][0 :1 ]
388+ l_prev , o_prev = _fixed_inner (qk , v_chunk , m_fix , l_scratch_ref [:, qsl ], o_scratch_ref [:, qsl ])
358389 l_scratch_ref [:, qsl ] = l_prev
359390 else :
360391 m_prev , l_prev , o_prev = _online_inner (
@@ -369,15 +400,7 @@ def _one_q_chunk(qs):
369400 # step stalls on instruction fetch -- measured 2.1x slower on pure compute
370401 # (91.2 -> 43.4 ms/layer), invisible in the static schedule, whose bundle
371402 # count and opcode mix stay exactly proportional to the work.
372- n_q = q_ref .shape [0 ] // q_sub
373- if n_q > 1 :
374- def _qbody (t , carry ):
375- _one_q_chunk (pl .multiple_of (t * q_sub , 128 ))
376- return carry
377- lax .fori_loop (0 , n_q , _qbody , None )
378- else :
379- for qs in range (0 , q_ref .shape [0 ], q_sub ):
380- _one_q_chunk (qs )
403+ _for_each_q_chunk (_one_q_chunk )
381404
382405 def compute_body_fixed (kv_compute_index , _ ):
383406 _step (kv_compute_index * bkv_compute , bkv_compute , True )
@@ -444,7 +467,6 @@ def _body_online():
444467 # head -- one body. Exact, since online continues from a fixed-m state:
445468 # m_next = max(m_prev, m_curr) and (o, l) rescale by exp2(m_prev - m_next).
446469 # Costs ~0.95 ms/layer vs two bodies, but avoids the ~3x cliff.
447- two_body_last = HYBRID_TWO_BODY_LAST or (q_ref .shape [0 ] // q_sub > 1 )
448470 if two_body_last :
449471
450472 @pl .when ((j == grid_width - 1 ) & is_fixed )
@@ -467,9 +489,13 @@ def _last_body():
467489 # ordered after this hop's compute.
468490 @pl .when (is_last_hop )
469491 def _write_out ():
470- l = l_scratch_ref [...]
471- l_inv = jnp .tile (1.0 / l , (head_dim_v_repeats , 1 ))
472- o_ref [...] = (o_scratch_ref [...] * l_inv ).astype (o_ref .dtype )
492+ def _write_out_chunk (qs ):
493+ qsl = pl .ds (qs , q_sub )
494+ l = l_scratch_ref [:, qsl ]
495+ l_inv = jnp .tile (1.0 / l , (head_dim_v_repeats , 1 ))
496+ o_ref [:, qsl ] = (o_scratch_ref [:, qsl ] * l_inv ).astype (o_ref .dtype )
497+
498+ _for_each_q_chunk (_write_out_chunk )
473499
474500
475501# Whether to single-buffer the q/out blocks when the q block spans the whole head.
@@ -486,6 +512,10 @@ def _write_out():
486512# instead of online for every head. Two bodies in the last block used to trigger the
487513# code-size cliff; with the q loop rolled the program may now fit.
488514HYBRID_TWO_BODY_LAST = False
515+ # Whether `mk[0, h]` carries the precomputed per-head fixed-m shift
516+ # `m_h = ceil(max_i ||q_i|| * max_j ||k_j||) - C` (Fix 2) instead of `max_j ||k_j||`.
517+ import os as _os
518+ PRECOMPUTE_FIXED_M = _os .environ .get ("IPERM_PRECOMPUTE_FIXED_M" , "1" ) == "1"
489519
490520def internal_ring_attention_forward (
491521 q : jax .Array ,
@@ -503,6 +533,7 @@ def internal_ring_attention_forward(
503533 uniform_fixed_m : bool = False ,
504534 mk : jax .Array | None = None ,
505535 fixed_m_recenter : float | None = None ,
536+ precomputed_m : bool | None = None ,
506537 q_sub_block : int | None = None ,
507538 use_experimental_scheduler : bool = False ,
508539 vmem_limit_bytes : int | None = None ,
@@ -579,6 +610,9 @@ def out_index_map(b, h, i, j, r, *_):
579610 pltpu .SemaphoreType .REGULAR ,
580611 ]
581612
613+ if precomputed_m is None :
614+ precomputed_m = PRECOMPUTE_FIXED_M
615+
582616 out = pl .pallas_call (
583617 functools .partial (
584618 _internal_ring_kernel ,
@@ -596,6 +630,7 @@ def out_index_map(b, h, i, j, r, *_):
596630 use_fixed_m = use_fixed_m ,
597631 uniform_fixed_m = uniform_fixed_m ,
598632 fixed_m_recenter = fixed_m_recenter ,
633+ precomputed_m = bool (precomputed_m ),
599634 q_sub = q_sub ,
600635 axis_names = tuple (axis_names ),
601636 ring_axis = ring_axis ,
@@ -640,6 +675,7 @@ def make_internal_ring_attention(
640675 use_fixed_m : bool = False ,
641676 uniform_fixed_m : bool = False ,
642677 fixed_m_recenter : float | None = None ,
678+ precomputed_m : bool | None = None ,
643679 q_sub_block : int | None = None ,
644680):
645681 """Batched `(b, h, s, d) -> (b, h, s, d)` callable. Deliberately NOT vmapped:
@@ -671,6 +707,7 @@ def _ring(q, k, v, mk=None):
671707 uniform_fixed_m = uniform_fixed_m ,
672708 mk = mk ,
673709 fixed_m_recenter = recenter ,
710+ precomputed_m = precomputed_m ,
674711 q_sub_block = q_sub_block ,
675712 use_experimental_scheduler = use_experimental_scheduler ,
676713 vmem_limit_bytes = vmem_limit_bytes ,
0 commit comments