forward(hidden[:, :-1], next_embeds): h = proj( cat([ h_norm(hidden), e_norm(next_embeds) ], dim=-1) ) h = h + block.attend(h) h = h + block.feed_forward(h).hidden return norm(h)
deflk_loss(target_logits: Tensor, draft_logits: Tensor) -> Tensor: """Eq. 16, at temperature 1.
Directly the negative log acceptance rate, with no ground-truth cross-entropy term -- optimising the thing that actually determines the speedup instead of a surrogate for it. """ p = torch.softmax(target_logits.float(), dim=-1) q = torch.softmax(draft_logits.float(), dim=-1) return -torch.log(acceptance_rate(p, q).clamp_min(1e-9)).mean()
返回:特征张量列表,每个形状 ``(B, T, C)``。 """ ifhasattr(state, "outputs"): # 通用兼容:直接给输出列表的对象 seq = list(state.outputs) elifhasattr(state, "sources"): seq = state.sources() elifisinstance(state, (list, tuple)): seq = list(state) else: raise TypeError(f"cannot read feature sources from {type(state)}") if picks isNone: return seq n = len(seq) idx = [i if i >= 0else n + i for i in picks] ifany(i >= n for i in idx) orany(i < 0for i in idx): raise IndexError(f"feature picks {picks} out of range for {n} sources") return [seq[i] for i in idx]
step_caches_snap = snapshot_caches(caches) # ① 打分前快照 for j inrange(draft_len): tok = draft_tokens[:, j:j+1] # ② 每步只喂 1 个 token lg = backend.score_token(tok) # ③ 走 caches 的增量前向 tgt_logits_list.append(lg[:, -1:])
# 逐 batch 求"连续接受前缀"长度:第一个 False 之前的 True 个数。 n_accepted_list = [] for b inrange(B): n = 0 for flag in accepted[b].tolist(): if flag: n += 1 else: break n_accepted_list.append(n) n_accepted = torch.tensor(n_accepted_list, device=target_logits.device) # (B,)
# 接受前缀的 token;长度取所有 batch 中最大接受数(不足由调用方按 n_accepted 截)。 max_acc = int(n_accepted.max().item()) if B else0 if max_acc > 0: accepted_tokens = draft_tokens[:, :max_acc] else: accepted_tokens = draft_tokens.new_zeros(B, 0)
# bonus token:在"被拒绝的位置"(= 接受前缀末尾)重采。 # 分布 = norm(max(p - q, 0))。若全部接受,则在最后一个草稿位置重采, # 分布取 p(目标)在该位置的分布。 bonus_token = torch.zeros(B, 1, dtype=torch.long, device=target_logits.device) for b inrange(B): nb = int(n_accepted[b].item()) if nb < L: pos = nb residual = torch.clamp(p[b, pos] - q[b, pos], min=0.0) # (V,) else: pos = L - 1 residual = p[b, pos] # 全部接受 total = residual.sum().clamp_min(1e-9) probs = residual / total if greedy: bonus_token[b, 0] = probs.argmax() else: bonus_token[b, 0] = torch.multinomial(probs, 1, generator=generator).item()
构造性证明(Model-Viz/src/parallel/moonep.py:110-190 的 CPU 实现):
初始化:每 rank 只持有 home 专家的 token
主循环:找欠载 rank u 和过载 rank o,从 o 搬 need = target - load(u) 个 token 到 u
终止:每轮填死一个 rank,至多 R-1 轮
2.5 拷贝权重
我们的问题是:在MoonEP真实场景下,拷贝权重不耗费时间嘛?
拷贝权重当然耗时而且是 ms 量级的真实开销,作者也把它算进了 benchmark。——MoonEP 的做法不是"不耗时",而是三件事:摊薄、隐藏、并且承认它。
README :25 自己有说明:
The comparison counts MoonEP's extra kernels: MoonEP adds planning and weight-prefetch kernels that DeepEP does not need, and they are already stacked in the bars above. Even with the whole critical path included, total dispatch time is on par with DeepEP v2's dispatch alone and pulls ahead under imbalance.
Rows [E, E+B): local prefetch slots, filled by buffer.prefetch_weight; ... Their physical memory comes from a process-global pool shared by all layers, so the extra cost is B expert weights per projection in total, not per layer.
Kept separate from dispatch so the plan-reuse path (combine bwd) can skip re-prefetching.
”
combine 的 backward 复用已存的 plan(README :144:"planning is skipped and no prefetch is needed")——反向传播不重复拷权重。所以一次前反向只拷一次,不是两次。
2.5.4 第三招:训练强制 B = E/R —— 用定理保证"拷贝量有上界"
README :56-59 这段是整个设计的收口:
Training: must use B = E/R — the planner duplicates experts from at most one remote home group per rank (≤ E/R experts), so every expert the group GEMM touches is local.
Inference (prefetch only, no gradients): B < E/R is allowed, B = 3–4 is recommended. If a rank ever needs more distinct remote experts than B, the group GEMM reads the overflow weights straight from the home rank through the symmetric mapping — slightly slower, with no impact on correctness.
”
两个信息量极大的点:
① 训练期 B = E/R 的理由正是那条定理。 "duplicates experts from at most one remote home group per rank" —— 这跟本项目 moonep.py:26-32 docstring 写的证明完全同源:
所以 B = E/R 不是拍脑袋的经验值,而是理论紧界。拷贝量因此有硬上界,不会随路由倾斜恶化——这正是 README :32 说的 "iteration time stays flat at every imbalance level"。
② 推理期 B 可以远小于界(3-4),代价是"溢出走 symmetric memory 直读"。 这是个漂亮的降级路径:拷不下的专家,group GEMM 直接通过对称内存映射读远端权重(memory semantics,靠 cu_seqlens 寻址),"slightly slower, with no impact on correctness"。推理没有梯度、B 可以砍到 1/8,拷贝成本随之暴跌。
零拷贝省掉的 epilogue:token 直接写到远端 rank 上的最终专家分组位置,"no permute in, no permute out",消掉了 "comm-buffer → user-buffer copy that dominates the epilogue"。这笔省下的量本身就很大。
# 预扫描:构建 exp_tab/slot_tab(active slot 的 compact 列表) n_active = 0 for b inrange(B): e = experts[b] if e >= 0: exp_tab[n_active] = e slot_tab[n_active] = b n_active += 1
Returns: 均衡的 RedundancyPlan。要求总指派数能被 R 整除 (Top-K × S × R 的算术让 target 恰为 S*K)。
算法(一步步验证界): 1. 初始:每个 rank 只持有它 home 专家的 token(``home = min(e // epr, R-1)``)。 2. 主循环: - 找第一个欠载 rank ``u``(load(u) < target); - 找当前最过载的 rank ``o``; - 从 o 的 holdings 中(从大专家开始,控制冗余副本数)取出 ``need = target - load(u)`` 个 token 搬到 u; - o 一定是够拿的(load(o) > target >= need),所以 u 恰好落 target; 3. 终止:某个 rank 一旦被填满再也不被触碰,至多 R-1 轮收敛。 """ ifisinstance(loads, Tensor): loads = [int(x) for x in loads.tolist()] n_experts = len(loads) experts_per_rank = experts_per_rank or -(-n_experts // n_ranks)
total = sum(loads) if total % n_ranks != 0: raise ValueError( f"总 token-专家指派数 {total} 不能被 {n_ranks} 个 rank 整除;" "MoonEP 的静态 shape 需要一个能整除的总量" ) target = total // n_ranks
plan = RedundancyPlan(n_experts, n_ranks, experts_per_rank) # 初始摆放:每家放自家专家的 token。 for e, load inenumerate(loads): if load: home = min(e // experts_per_rank, n_ranks - 1) plan.holdings[home][e] = plan.holdings[home].get(e, 0) + load
current = [sum(h.values()) for h in plan.holdings]
whileTrue: # 找到第一个欠载 rank——它的负载将被精确补齐。 under = next((r for r inrange(n_ranks) if current[r] < target), None) if under isNone: break# 都已均衡(含整除舍入内) # 找最过载的 rank 作为"供应者"。 over = max(range(n_ranks), key=lambda r: current[r]) if current[over] <= target: break# 整数舍入内已均衡 need = target - current[under]
# 从 over 手里按"大专家优先"搬 need 个 token 给 under。 # 大专家优先 => 尽可能少搬不同的专家 => 冗余副本数少。 for expert insorted(plan.holdings[over], key=lambda e: -plan.holdings[over][e]): if need == 0: break take = min(plan.holdings[over][expert], need) plan.holdings[over][expert] -= take if plan.holdings[over][expert] == 0: del plan.holdings[over][expert] plan.holdings[under][expert] = plan.holdings[under].get(expert, 0) + take # 非本地专家级别才计入冗余。 if expert // experts_per_rank != under: plan.redundant[under].add(expert) need -= take
current = [sum(h.values()) for h in plan.holdings] plan.moves += 1 if plan.moves > n_ranks * n_ranks: # 保险丝:理论上不会触发 raise RuntimeError("MoonEP 规划器未收敛")
# 每个 (rank, expert) 的剩余容量,直接来自 plan。 remaining = [dict(h) for h in plan.holdings] dest = torch.empty_like(flat) cursor = {} for idx inrange(flat.numel()): e = int(flat[idx]) r = cursor.get(e) if r isNoneor remaining[r].get(e, 0) == 0: r = next( (rr for rr inrange(self.n_ranks) if remaining[rr].get(e, 0) > 0), None, ) if r isNone: raise RuntimeError(f"专家 {e} 已无容量") cursor[e] = r remaining[r][e] -= 1 dest[idx] = r
for d inrange(R): # :149 遍历每个 rank local_start = d * epn local_end = local_start + epn
# ① 筛出"该 rank 要算、但不是自己 home 的"专家 = 远端专家 remote_experts = [ e for e inrange(E) if alloc[e, d].item() > 0andnot (local_start <= e < local_end) ] # :153-156
# ② 按 token 数降序排(tie-break 用 expert id 保证确定性) remote_experts.sort(key=lambda e: (alloc[e, d].item(), e), reverse=True) # :160
# ③ 只取前 B 个,写进 experts_to_copy for b, e inenumerate(remote_experts[:B]): # :162 experts_to_copy[d, b] = e # :163 ★决策落盘 remote_stats_all[e // epn, 1] += 1
决策规则三句话:
候选:alloc[e,d] > 0 且 e 不属于 rank d 的 home 区间 → 远端专家(:153-156)
排序:按该 rank 要为它算的 token 数降序(:160),注释 :158-159 说明动机:"Pick the B remote expert segments with the most tokens for VM prefetch"
紧接着 :166-193 用这个决策重排物理 token 顺序:VM group id 0..E-1 是全局专家组,E..E+B-1 是被选中远端专家的 prefetch slot(注释 :169-171);:177if g not in experts_to_prefetch 确保被 prefetch 的专家不在原位重复占空间。
under = next((r for r in range(n_ranks) if current[r] < target), None) # :161 欠载 rank over = max(range(n_ranks), key=lambda r: current[r]) # :165 最过载 rank need = target - current[under] # :168 精确补齐量
Width 分离:常规 MoE 中所有专家都是全宽(d 维)的,专家越多参数量和计算量线性放大。LatentMoE 在路由专家前加一个 W_down 投影,把 token 从 d 维降到 l 维(l < d),路由专家全部在 l 维工作,最后 W_up 再升回 d 维。这样路由专家的参数和计算量都从 d 级降到 l 级,K3 中 l=3584、d=4096,约节省 12.4% 的专家侧参数。
classGroupedExperts(nn.Module): """``n`` SiTU-GLU experts stored as batched weights.
Keeping the experts as three stacked tensors rather than ``n`` modules is what lets dispatch become a grouped GEMM -- the shape MoonEP schedules in phase 5. """
# 【中文释义】GroupedExperts 把 n_routed 个专家"打包"成三块叠加权重张量: # - gate:(E, l, H) —— 每个专家对潜向量做线性变换,作为 SiTU 门控的输入 # - up: (E, l, H) —— 同源上分支,同样在潜空间 l 中 # - down:(E, H, l) —— 把专家内部宽度 H 投影回潜宽度 l # 不把每个专家实现为独立 nn.Module 的目的:dispatch 可以变成「一次 grouped GEMM」 # 的形状(一次 matmul 覆盖所有被选中的专家),这正是 MoonEP(§5)在 EP 分布式 # 下调度 S×K token 到专家时使用的组织方式。CPU 教学实现里 forward_indexed 用 # 逐专家稠密 matmul + 排列替代,语义一致。 def__init__( self, n_experts: int, dim_in: int, hidden: int, dim_out: int, *, beta_gate: float, beta_up: float, ): super().__init__() self.n_experts = n_experts self.dim_in, self.hidden, self.dim_out = dim_in, hidden, dim_out self.beta_gate, self.beta_up = beta_gate, beta_up self.gate = nn.Parameter(torch.empty(n_experts, dim_in, hidden)) self.up = nn.Parameter(torch.empty(n_experts, dim_in, hidden)) self.down = nn.Parameter(torch.empty(n_experts, hidden, dim_out)) #: Quantisation-aware training (paper section 4.1.4). When enabled the #: forward pass sees MXFP4 expert weights and MXFP8 activations while the #: master weights stay in higher precision, so the model adapts to the #: deployment numerics instead of meeting them for the first time at #: export. Off by default; ``kimi3.quant.mxfp.apply_qat`` turns it on. self.qat_enabled = False self.reset_parameters()
defforward_indexed(self, x: Tensor, expert_ids: Tensor) -> Tensor: """Apply expert ``expert_ids[t]`` to row ``t`` of ``x``.
Rows are permuted into expert-contiguous order so each expert's work is a single dense matmul, then permuted back once. This is the shape MoonEP turns into a grouped GEMM with a fused permute/unpermute in phase 5; here the permutation is a plain gather and the inverse a single scatter. """ # 【中文释义】按 expert_ids 把 (tokens, l) 的 token 行"按专家分组连续化": # - order = 按专家 id 排序的索引(stable sort 保证同专家 token 保持输入相对顺序) # - counts = 每个专家获得的 token 数 # - offsets = 每段专家的起始位置(前缀和),[0, c0, c0+c1, ...] # 之后每个专家直接取 x_sorted[lo:hi] 一整段做一次稠密 matmul + SiTU-GLU, # 相当于把它变成一个 grouped GEMM 的一次性切分;最后用 index_copy 把各段 # 输出按 order 回填到原始 token 顺序。这一步对应论文 §5 / MoonEP 中 # "fused permute/unpermute" 的形状——此处是朴素 gather + scatter 的 CPU 实现。 from .situ_glu import situ_glu
defforward_all(self, x: Tensor) -> Tensor: """Apply every expert to every row: ``(n, tokens, dim_out)``.
Only used by tiny configs and by tests; the dispatch path is the norm. """ # 【中文释义】把所有专家对每个 token 的投影一次性算出,返回 (n, tokens, l)。 # 仅用于小配置与测试(test_grouped_dispatch_matches_dense_evaluation 用它当 # 稠密 oracle),正式 dispatch 永远走 forward_indexed。 from .situ_glu import situ_glu
classStableLatentMoE(nn.Module): # 【中文释义】Stable LatentMoE 的完整组件:把一个 dim 维输入 token 通过潜空间 # (latent width l)路由到 top_k 个专家 + 全宽 shared 专家,汇聚后回投影到 d 维。 # 全部宽:dim d → 潜 l → 专家内部 hidden H → 潜 l → dim d。 # 与 kimi3-master/kimi3/modules/moe.py 逐行镜像;K3 真实配置: # n_routed=896, n_shared=2, top_k=16(稀疏度=56), latent=3584, # expert_hidden=shared_hidden=3072, SiTU beta_gate=4.0, beta_up=25.0, # QB enabled/bins=1000/ema=0.0(见 config.py MoEConfig 默认值)。 def__init__(self, dim: int, cfg: MoEConfig, *, eps: float = 1e-6): super().__init__() self.dim = dim self.cfg = cfg ell = cfg.latent_dim
self.down_proj = nn.Linear(dim, ell, bias=False) # W_down: d -> l self.up_proj = nn.Linear(ell, dim, bias=False) # W_up: l -> d # The stabiliser of section 2.3.1, between aggregation and up-projection. self.latent_norm = RMSNorm(ell, eps) if cfg.rmsnorm_before_up else nn.Identity()
K3 routes with a per-expert bias added to the score used for Top-k selection but *excluded* from the mixture weights::
s_i = sigmoid(W_r x_i) T_i = argtopk_k(s_i + b) selection sees the bias p_ij = s_ij / sum_{r in T_i} s_ir gradients do not
so ``b`` steers dispatch without distorting the optimisation of the router.
The question is how to set ``b``. The usual loss-free rule nudges it by a fixed step in the direction of the load error, ``b <- b + gamma * sign(target - load)``, whose step size is a hyperparameter and whose equilibration is slow -- and at 896 experts per layer, slow equilibration means persistently imbalanced expert-parallel training and undertrained experts.
Quantile Balancing instead jumps straight to the bias that *makes* each expert's load equal to its target, in one forward pass. Routing uses Top-(k+1) rather than Top-k on the biased score: the first ``k`` entries are the routes actually taken, and the ``(k+1)``-th is the cutoff ``alpha_i`` an expert must exceed to enter token ``i``'s Top-k. Taking the cutoff from the routing itself avoids a separate token-side quantile. With the cutoffs fixed, the count of tokens routed to expert ``j`` under a candidate bias is monotone in the threshold, so setting that count to the target ``q = mk/n`` makes the bias the ``(q+1)``-th largest margin ``s_ij - alpha_i``. That is Eq. 14::
bhat_j <- -quantile_{1-k/n}(s_{:,j} - alpha) b <- bhat - mean(bhat)
Appendix C derives this as the exact coordinate minimiser of the dual of the maximum-score balanced-assignment LP, which is why it needs no learning rate.
The update is applied on the *next* step -- a batch is never routed with a bias derived from itself -- and the bias is frozen at inference.
Histogram estimation (appendix D) --------------------------------- At scale the quantile spans millions of margins sharded across data-parallel ranks and gradient-accumulation steps, so gathering them exactly is not viable. Instead each rank histograms the *required bias* ``r_ij = alpha_i - s_ij`` -- the bias that would place expert ``j`` exactly at token ``i``'s cutoff -- into per-expert bins. Counts are additive, so a single integer all-reduce yields the histogram of the pooled global batch, and the estimate is the true global quantile up to the bin width, not an average of per-rank quantiles. Cost is ``n * B`` integers per layer per step, independent of the token count. """
# Buffers, not parameters: the bias is set by the balancing rule, never by # a gradient, and it is frozen at inference. self.register_buffer("bias", torch.zeros(n_experts)) self.register_buffer("histogram", torch.zeros(n_experts, bins, dtype=torch.long)) self.register_buffer("hist_lo", torch.zeros(())) self.register_buffer("hist_width", torch.zeros(())) self.register_buffer("token_count", torch.zeros((), dtype=torch.long)) self.register_buffer("load", torch.zeros(n_experts, dtype=torch.long))
@torch.no_grad() defupdate_bias(self, group=None) -> Tensor: """Apply Eq. 14 from the pooled histogram. Call once per optimizer step.
Returns the new bias. The single ``n * bins`` integer all-reduce here is the entire communication cost of load balancing. """ # 【中文释义】按直方图估算出的分位数更新偏置(Eq. 14),通常每 optimize step # 调用一次。流程: # 1) all-reduce 直方图与 token 计数(如果多卡):偏置只做负载均衡用,通信量 # 就是 n*bins 个整数 / (层, 步); # 2) 目标负载 q = m·k/n(m 个 token,每个选 k 个专家,平均到 n 个专家); # 3) 沿直方图求第 q 大的分箱:对每个专家找"累计计数首次 ≥ q"的 bin, # 再在 bin 内按比例插值得到分位数估计: # b_hat_j = hist_lo + (bin_idx + frac) * hist_width # 这里 frac 是 (q - before)/counts(bin 已排到那一格后的位置); # 4) 减去均值再 EMA 平滑,写回 bias(Eq. 14 的 mean-centre 处理保证 # sum(b)=0,尺度可控)。 ifnotself.enabled orint(self.token_count) == 0: returnself.bias
hist = self.histogram total = self.token_count.clone() if dist.is_available() and dist.is_initialized(): dist.all_reduce(hist, group=group) dist.all_reduce(total, group=group)
m = int(total) q = m * self.top_k / self.n_experts # target load per expert cum = hist.cumsum(dim=1) target = torch.tensor(q, device=hist.device).ceil()
Full 形式 (Eqs. 8‑9):每层输出都保留为独立 source,算术 O (L²d)、内存 O (Ld)
Block 形式 (Eq. 10):把 L 层切成 N 块,每块塌缩成单个求和表示 b_n,内存降到 O (Nd)
退化关系: block_units=1 时 Block 自动退化为 Full
1.2 状态机 API
state = attnres.init_state(embedding) # b_0 = h_1 for i, layer in enumerate(layers): h = attnres.mix(state, i) # 深度注意力混合 out = layer(h) # 子层变换 state = attnres.add_output(state, out) # 块归档 out = attnres.finalize(state) # 最终聚合
0x02 Tab 1: Demo 四联图
2.1 目的
通过一个 copy 任务的训练过程,展示 AttnRes 的核心行为:训练前 alpha 均匀(等价标准残差),训练后 alpha 学会重点读 embedding;Block 形式内存有界 vs Full 形式线性增长;AttnRes vs 标准残差的训练收敛速度对比。
2.2 Copy 任务是什么
Copy 任务定义:给定随机输入 x, 模型输出必须逼近 x 本身(即 target = input)。损失函数为 MSE。
K3):K3 技术报告 §2.2 明确 "we partition its layers into 8 blocks with 12-layer size, giving a partial final block and 9 total blocks when counting the embedding layer"。K3 总层数为 93 层。
增加:每层一次 w_l·RMSNorm(v_i) 是 d 点积 × n 源 = O(nd);总 L 层 O(nLd)。
n 个源:Full n=L(K3 以 sublayer 计 2L);Block n=N+1。
内存开销:每层存 w_l∈R^d,各 key&value 来自前存输出,零新增。
2.1.5 参数敏感性
w_l 的初始值:论文 §5 明确"所有伪查询向量必须初始化为零"——这保证初始 w=0 → α=uniform(softmax(0)=均匀) → 初始状态 = 标准残差(等权平均)!论文原文:"This reduces AttnRes to an equal-weight average at the start of training and prevents training volatility, as we validated empirically." 本项目 queries 用 torch.zeros(...) 初始化正是此设计。
论文经验:N≈8「recover most of the benefit」;增加 block count 一般会提升但受限于通信/内存。
2.3.6 工程实现(论文 Fig. 2 伪代码 vs 本项目)
论文 Fig. 2 的关键伪代码:
defblock_attn_res(blocks, partial_block, proj, norm) -> Tensor: # blocks: N tensors [B,T,D]; partial_block: [B,T,D] (intra-block partial sum) V = torch.stack(blocks + [partial_block]) # [N+1, B, T, D] K = norm(V) logits = torch.einsum('d, n b t d -> n b t', proj.weight.squeeze(), K) h = torch.einsum('n b t, n b t d -> b t d', logits.softmax(0), V) return h
state = attn_res.init_state(embedding) for i, layer in enumerate(layers): h = attn_res.mix(state, i) out = layer(h) state = attn_res.add_output(state, out) if i % 12 == 0: # 每块结束看一次 alpha = attn_res.attention_weights(state, i) print(f"layer {i}: alpha over {len(state.sources())} sources = {alpha}")
没有这个方法,你只能看 mix() 的最终输出,完全不知道模型在深度维上"读"了哪些层。
总结:状态机的 4 个方法对应 4 个问题
方法
行号
解决什么问题
关键操作
init_state()
151-155
初始化(embedding 永远是源)
blocks=[embedding]
mix()
157-176
选择性(softmax 深度注意)
queries[unit_idx]
× RMSNorm × softmax
add_output()
178-195
内存(块塌缩 O(Ld)→O(Nd))
满 block_units 时归档 partial
finalize()
197-208
正确性(兜底尾巴)
不满块也归档 + 最终聚合
attention_weights()
210-221
可解释性(暴露 alpha)
只打分不混合
Block 和 Full 的等价性(attn_res.py:230-244):FullAttentionResiduals 就是 block_units=1——每个输出立即归档成"块",数学上和 Block 完全等价,只是内存策略不同。这证明了块塌缩不损失表达能力,只是工程上的内存优化。
# 单机视角(你想象的) state = attn_res.init_state(emb) for layer in all_96_layers: h = attn_res.mix(state, i) out = layer(h) state = attn_res.add_output(state, out) # blocks 累积到 9 个 final = attn_res.finalize(state)
这些方法都停留在递推范式内。AttnRes 更进一步:用直接跨层注意替换深度递推——正如 Transformer 用 self-attention 替换时间递推。论文明确:"Since the number of layers in current architectures remains well within the practical regime of softmax attention, we adopt vanilla depth-wise attention",并把"更高效(线性复杂度)替代"列为 future work。
既然序列建模又被 linear attention(Mamba、GLA、DeltaNet 等)挑战,深度维度是否也有对应的"更高效 linear-complexity 深度注意"?论文明确说"incorporating more expressive yet memory-efficient (e.g. linear-complexity) alternatives is a natural direction"。
A standard residual stream compresses everything a network has computed into one vector per position and hands it to the next layer -- a depth-wise bottleneck directly analogous to the hidden state of an RNN over time. Attention Residuals apply the transformer's own fix to depth: layer ``l`` *attends* over the outputs of all preceding layers instead of accumulating them uniformly.
q_l = w_l learned pseudo-query, per layer k_i = v_i = f_i(h_i), k_0 = v_0 = h_1 the embedding is always a source phi(q, k) = exp(q^T RMSNorm(k)) RMSNorm stops a layer with large alpha_{i->l} = phi(q_l, k_i) / sum_j phi outputs from dominating h_l = sum_i alpha_{i->l} v_i Eq. 9
Note what this replaces: ``h_l`` is the mixture itself, not "previous state plus something". Uniform accumulation is one particular setting of ``alpha``.
The full form costs ``O(L^2 d)`` arithmetic -- affordable at ``L < 100`` -- but ``O(Ld)`` memory, which is not, once every layer output must stay resident across pipeline stages. The block form fixes that: partition ``L`` layers into ``N`` blocks, reduce each block to a single summed representation ``b_n``, and attend over block representations plus the running partial sum of the current block. Memory drops from ``O(Ld)`` to ``O(Nd)``. For K3, ``N = 8`` blocks of 12 layers (the last partial), giving 9 sources once the embedding is counted.
@dataclass classAttnResState: """Sources visible to the next sublayer.
``blocks[0]`` is the token embedding (``b_0 = h_1``), followed by one entry per completed block. ``partial`` is the running sum inside the current block.
defdetach(self) -> "AttnResState": """Detach a deep copy; used where gradients must not flow through history.""" return AttnResState( blocks=[b.detach() for b inself.blocks], partial=Noneifself.partial isNoneelseself.partial.detach(), units_in_block=self.units_in_block, )
classAttentionResiduals(nn.Module): """Owns the per-unit pseudo-queries and the block bookkeeping.
``n_units`` counts the addressing units: sublayers when ``granularity="sublayer"`` (Figure 2 draws a residual arrow per module), or decoder layers otherwise. ``block_units`` is the block size measured in the same unit.
【中文释义】参数初始化:``queries`` 形状为 ``(n_units + 1, dim)``——每个 unit 一个伪查询,**多出的一个**供 ``finalize`` 时在所有块表示上做最终聚合; ``key_norm_weight`` 是 RMSNorm 的逐维缩放权重。 """ super().__init__() self.dim = dim self.n_units = n_units self.block_units = block_units self.eps = eps self.full = full # One learned pseudo-query per unit, plus one extra for the final # aggregation over all block representations. # 每个 unit 分配一个可学习的伪查询,外加一个给 finalize 的最终聚合项。 self.queries = nn.Parameter(torch.zeros(n_units + 1, dim)) self.key_norm_weight = nn.Parameter(torch.ones(dim))
definit_state(self, embedding: Tensor) -> AttnResState: """``b_0 = h_1``: the token embedding is always an available source.""" # 【中文释义】状态初始化:block 0 固定为 token embedding,保证 embedding # 永远是第一个可读源;partial 为空、计数为 0。 return AttnResState(blocks=[embedding], partial=None, units_in_block=0)
defmix(self, state: AttnResState, unit_idx: int) -> Tensor: """Eq. 9 -- the representation handed to unit ``unit_idx``.
defadd_output(self, state: AttnResState, output: Tensor) -> AttnResState: """Fold a unit's output into the current block's partial sum, rolling the block over when it is full.
classFullAttentionResiduals(AttentionResiduals): """The unblocked ``O(L^2 d)`` form of Eqs. 8-9, used as a test oracle.
Every unit output is kept as its own source, so ``block_units = 1`` recovers this behaviour from the block implementation -- which is exactly the equivalence the test suite asserts.
S = S_0.clone() for i in 0..n_accepted‑1: out_i, S = kda_step(q[i], k[i], v[i], g[i], beta[i], S) output_stacked = stack(out_0, ..., out_{n‑1}) # (B, H, n, Dv)
「MHA 也能做成池,分页不稀罕。」 对,分页本身不稀罕。但关键是:这个池必须同时装得下「随 T 增长的 token」和「尺寸对 T 不敏感的整块状态」——这是 MLA+KDA 的独特混合,普通的 KV 桶做不到。PageKind 的双值枚举正是「两种物理性质共存于一个分配池」的体现。
「为什么 pack_heads 要保持 head 连续,直接 flatten 不行吗?」 因为 head 是跨节点传输的最小语义单元。未来的 KCP(上下文并行)按 head 拆分通信;如果一个 head 的字节被切到不同页/不同段,传输时就要额外拼装。保持 head 连续 = 每个 head 的字节流「自包含」,可独立搬运。
4.7 何时用 / 何时不用
何时用:任何要同时管理「随 T 增长的 KV」和「固定尺寸状态」的 K3 混合 serving。这是底层,上面所有组件都建立在它之上。
为什么需要"必须共存"这个约束:K3 的递归状态是逐层的(每层有自己的 S_l)。要恢复"整个 block"在边界 B 的状态以继续算,每一层在 B 的 checkpoint 都得在——少一层,算出的下一 token 就是错的。所以"一起存在才算数"是硬约束。group 就是把"哪几个东西必须一起存在"打包成一个抽象单元的机制。
5.5.1 先看清 K3 的层排布(真实 layer_schedule())
deflayer_schedule(self): body = self.n_layers - 1 period = 4 schedule = [] for i inrange(body): schedule.append("mla"if (i + 1) % period == 0else"kda") schedule.append("mla") # 尾部再挂一个全局 MLA return schedule
ReplaySSM spec-verify: fold-every-commit — a per-slot raw-input window replaces the recurrent verify's per-draft full-state snapshots. The verify stores each draft step's raw inputs into a per-slot window and the commit replays the accepted prefix into the fp32 checkpoint.
ReplaySSM-KDA: the accepted drafts live in the per-slot ring (written during verify); no intermediate_ssm is allocated. Replay the accepted prefix into temporal instead of scattering an intermediate state.
注释里写得很清楚:"Primarily a GDN (scalar-gate) decode-bandwidth optimization (~1.2-1.5x at batch >= 64)"。batch 大时状态矩阵的 HBM 往返是瓶颈,raw input 环能把带宽砍一个数量级。
KDA 的坑:注释也说了——KDA uses its selected Triton or Helion implementation, but its per-K gate ring is larger and ReplaySSM is typically slower than packed KDA decode。KDA 的 gate 是 per-channel 向量(H·dk),比 GDN 的标量 gate 大得多,所以 ring 反而更占带宽,通常更慢,要 benchmark 后再开。
乘上 H 是因为每个 Token 有 H 个 head,每个 head 都各自保存一组q, k, g, v, beta。RawInputRing 的五个 list 正好对应上表:_q, _k, _g, _v, _beta。它们按相同的时间顺序追加和淘汰,所以第 i 个位置的五个张量必须视为一个不可拆分的投影组。
state = g.exp().unsqueeze(-1) * state sk = torch.einsum("bhd,bhdv->bhv", k, state) delta = beta.unsqueeze(-1) * (v - sk) state = state + k.unsqueeze(-1) * delta.unsqueeze(-2) out = torch.einsum("bhd,bhdv->bhv", q, state)
即:段 i+1 的入态 = 「段 i 过渡到它自身」+ 「段 i 从零状态自主生成的状态」。 这正是 delta rule 与 vanilla 线性注意力的根本差异所在 —— 状态不直接可加, 必须用一个逐段不同的仿射变换来传递。 """ n = transitions.shape[0] incoming = torch.zeros_like(zero_states) state = torch.zeros_like(zero_states[0]) for i inrange(n): incoming[i] = state state = transitions[i] @ state + zero_states[i] return incoming
# 原文注释: #: Number of independent KDA cache groups a checkpoint must exist in. n_kda_groups: int = 1 #: ``group -> set of token boundaries with a persisted KDA checkpoint``. kda_checkpoints: dict[int, set[int]]
常见疑问:recurrent state 和 convolution state 是什么?SGLang 有对应吗?
Recurrent state 是 KDA 长期递推矩阵,携带压缩后的长历史;convolution state 是短卷积需要的最近窗口,携带局部连续性。两者可以采用不同存储策略:temporal state 可压缩,局部窗口通常保留原精度。SGLang 的 KDA‑aware checkpoint 主要围绕 recurrent state 展开,但合法恢复不能只恢复它;对应局部窗口状态也要恢复或按明确规则重建。
因此 "统一" 主要指请求调度和生命周期协调;底层仍承认 KV 与 KDA state 的语义差异。
7.1 State block 是什么
State block 是物理内存管理单位,不等于一个 token,也不等于一个完整请求:
State block B ┌─────────────────────────────┐ │ 一段连续执行产生的 KDA state │ │ recurrent state │ │ convolution state │ │ 边界/版本/布局等元数据 │ └─────────────────────────────┘
Fine‑grained Partial Hit 的做法是:仍然使用这个 128‑token 物理 block,但在其中记录多个更细的 Prefix Entry:
State block B: entry at 32 -> state after token 32 entry at 64 -> state after token 64 entry at 96 -> state after token 96 entry at 128 -> state after token 128