[{"data":1,"prerenderedAt":384},["ShallowReactive",2],{"blog-27688v1":3},{"id":4,"title":5,"body":6,"date":372,"description":373,"extension":374,"image":375,"meta":376,"navigation":377,"path":378,"seo":379,"sitemap":380,"stem":381,"tags":382,"__hash__":383},"blog/blog/2510.27688v1.md","連續自迴歸語言模型 CALM 技術解析",{"type":7,"value":8,"toc":359},"minimark",[9,13,25,33,39,45,50,53,56,72,78,82,88,94,99,161,170,174,198,218,226,240,244,247,279,283,309,313,336,351],[10,11,5],"h1",{"id":12},"連續自迴歸語言模型-calm-技術解析",[14,15,16,20,21,24],"p",{},[17,18,19],"strong",{},"研究單位","：騰訊微信 AI × 清華大學\n",[17,22,23],{},"核心命題","：傳統大模型逐字（Token-by-token）生成的效率已觸及物理天花板。CALM 將離散 Token 轉換為連續向量（Continuous Vectors），將生成步數縮減至原來的 1/K 為大語言模型的擴充開闢了全新的效率路徑。",[14,26,27],{},[28,29,30],"a",{"href":30,"rel":31},"https://arxiv.org/html/2510.27688v1",[32],"nofollow",[14,34,35],{},[28,36,37],{"href":37,"rel":38},"https://shaochenze.github.io/blog/2025/CALM/",[32],[14,40,41],{},[28,42,43],{"href":43,"rel":44},"https://github.com/mahmoodlab/clam",[32],[46,47,49],"h2",{"id":48},"一背景與動機離散生成的效率瓶頸","一、背景與動機：離散生成的效率瓶頸",[14,51,52],{},"現有的大型語言模型（LLMs）都依賴「逐字預測」的舊範式。每個 Token 都需要進行一次完整的前向計算，當序列變長時，計算成本呈線性甚至平方級增長。",[14,54,55],{},"儘管文本表徵已經從字符集進化到子詞集（如 BPE），但這種「離散表示」已觸及物理極限：",[57,58,59,66],"ul",{},[60,61,62,65],"li",{},[17,63,64],{},"詞表膨脹的詛咒","：為了涵蓋更多語義，詞表規模從 32,000 膨脹至 256,000，導致最後一層的計算與記憶體成本飆升。",[60,67,68,71],{},[17,69,70],{},"資訊密度極低","：每個離散詞元僅攜帶約 15–18 bit 的資訊，潛力已然耗盡。模型擁有龐大的算力，卻被困在一次只能預測一個低資訊量 Token 的勞力密集工作中。",[14,73,74,77],{},[17,75,76],{},"結論","：必須轉向高維連續路線，從根本上改變生成機制。",[46,79,81],{"id":80},"二calm-核心思路與原理","二、CALM 核心思路與原理",[14,83,84,87],{},[17,85,86],{},"基本機制","：利用自編碼器(Autoencoder)將 K 個離散 Token 壓縮為「一個」高維連續向量。這使得模型的序列長度從 T 降至 T/K，生成步數直接縮減為原來的 K 分之一。",[14,89,90,93],{},[17,91,92],{},"本質","：以更高的「語義頻寬」(Semantic Bandwidth)換取更少的生成步驟，實現算力與速度的跨越式突破。",[95,96,98],"h3",{"id":97},"傳統範式-vs-calm-連續模型","傳統範式 vs. CALM 連續模型",[100,101,102,118],"table",{},[103,104,105],"thead",{},[106,107,108,112,115],"tr",{},[109,110,111],"th",{},"維度",[109,113,114],{},"傳統離散模型 (Discrete)",[109,116,117],{},"CALM 連續模型 (Continuous)",[119,120,121,135,148],"tbody",{},[106,122,123,129,132],{},[124,125,126],"td",{},[17,127,128],{},"擴容方式",[124,130,131],{},"詞表指數級膨脹，算力爆炸",[124,133,134],{},"增加向量維度，算力平滑增長",[106,136,137,142,145],{},[124,138,139],{},[17,140,141],{},"計算目標",[124,143,144],{},"從數十萬詞中選一個機率",[124,146,147],{},"在連續空間中直接預測精確座標",[106,149,150,155,158],{},[124,151,152],{},[17,153,154],{},"序列長度",[124,156,157],{},"極長序列 T",[124,159,160],{},"短序列 T/K，步數銳減",[162,163,164],"blockquote",{},[14,165,166,169],{},[17,167,168],{},"通俗比喻","：傳統生成就像在「擁有數十萬個抽屜的倉庫中盲目尋寶」，而 CALM 則是「直接繪製高維座標」，徹底擺脫了詞表膨脹的物理限制。",[46,171,173],{"id":172},"三calm-的四大技術支柱","三、CALM 的四大技術支柱",[175,176,177,183,193],"ol",{},[60,178,179,182],{},[17,180,181],{},"高保真自編碼器 (Autoencoder)","\n將 $K$ 個離散詞元壓縮為 128 或更高維度的連續向量。在約 7500 萬參數的輕量級開銷下，重構準確率高達 > 99.9%。它是實現離散與連續間無損轉換的基石。",[60,184,185,188,189,192],{},[17,186,187],{},"能量生成頭與無似然框架","\n摒棄了傳統的 Softmax 機率計算，在連續空間中直接透過迴歸（Regression）預測下一個目標向量，無需像擴散模型（Diffusion）那樣進行數十上百次的去噪迭代，實現",[17,190,191],{},"單步生成","。",[60,194,195],{},[17,196,197],{},"連續空間魯棒性三大魔法",[57,199,200,206,212],{},[60,201,202,205],{},[17,203,204],{},"變分正則化 (Variational Regularization)","：熨平潛空間（Latent space）中的褶皺與斷層。",[60,207,208,211],{},[17,209,210],{},"KL 裁減 (KL Clipping)","：防止潛空間維度嚴重塌陷。",[60,213,214,217],{},[17,215,216],{},"雙重 Dropout","：提供額外的抗噪能力。\n這三者共同構建了平滑且富有彈性的高維地形，確保微小的生成偏差不會導致亂碼。",[175,219,221],{"start":220},4,[60,222,223],{},[17,224,225],{},"無機率評估體系 BrierLM & 溫度控制",[57,227,228,234],{},[60,229,230,233],{},[17,231,232],{},"Brier 評估 (Brier Score)","：傳統的困惑度 (Perplexity) 在連續模型中失效。CALM 引入了無似然的 Brier 分數（準確率 − 碰撞率）作為專屬標尺，與交叉熵損失呈極強負相關。",[60,235,236,239],{},[17,237,238],{},"黑盒溫度採樣","：因為沒有傳統機率分母可調，CALM 以「批量大小 N」(Batch Size) 作為新型的溫度旋鈕。N 小則低溫高確定性，N 大則高溫高多樣性。搭配拒絕採樣機制，效果與傳統溫度調節高度一致。",[46,241,243],{"id":242},"四流程總覽","四、流程總覽",[14,245,246],{},"整個運作閉環如下，速度遠超傳統的逐字生成：",[175,248,249,255,261,267,273],{},[60,250,251,254],{},[17,252,253],{},"輸入","：離散 Token 序列。",[60,256,257,260],{},[17,258,259],{},"壓縮","：編碼器將每 K 個 Token 壓縮為 1 個連續向量。",[60,262,263,266],{},[17,264,265],{},"處理","：Transformer 骨幹網絡在極短的連續向量序列上進行深層注意力計算。",[60,268,269,272],{},[17,270,271],{},"預測","：生成頭直接單步預測出下一個目標連續向量。",[60,274,275,278],{},[17,276,277],{},"還原","：解碼器將預測出的連續向量無損還原為 K 個離散 Token。",[46,280,282],{"id":281},"五實驗結果與性能優勢","五、實驗結果與性能優勢",[57,284,285,291,297,303],{},[60,286,287,290],{},[17,288,289],{},"極致的算力節省","：在 K=4 的設置下，訓練算力降低 44%，推理算力降低 34%，生成步數從 T 驟降至 T/4。",[60,292,293,296],{},[17,294,295],{},"打破「更大必定更耗算力」的認知","：CALM 參數達 3.71 億（略大於對照組），卻能在打平甚至超越基線性能的情況下，消耗更少的整體算力。",[60,298,299,302],{},[17,300,301],{},"全新縮放維度「資訊密度」","：引入了全新的 Scaling 維度。當語義頻寬提高時，CALM 能以更低的算力達到更高的 Brier 分數，擊穿了原有的算力-性能邊界。",[60,304,305,308],{},[17,306,307],{},"「後積薄發」的訓練動態","：呈現特殊的訓練曲線。模型在 20 萬步後性能會迎來急劇上升並超越傳統模型，顯示掌握高維連續規律需要前期積累，但後續爆發力極強。",[46,310,312],{"id":311},"六總結與展望","六、總結與展望",[14,314,315,316,319,320,323,324,327,328,331,332,335],{},"CALM 成功構建了從離散到連續的完整基礎設施，其組件協同形成了一個強大的生態閉環：\n",[17,317,318],{},"魯棒自編碼器"," (高保真壓縮) → ",[17,321,322],{},"頻寬倍增"," (步數縮短至 1/K) → ",[17,325,326],{},"單步生成引擎"," (極速推理) → ",[17,329,330],{},"拒絕採樣"," (多樣性可調) → ",[17,333,334],{},"BrierLM"," (公正尺標)。",[14,337,338,339,342,343,346,347,350],{},"這項研究證明了語言模型的未來可能不在離散的像素與詞表中，而是在連續的波瀾裡。這為未來的",[17,340,341],{},"多模態連續生成","、",[17,344,345],{},"強化學習整合","以及",[17,348,349],{},"終端設備部署","開啟了全新的紀元。",[162,352,353],{},[14,354,355,358],{},[17,356,357],{},"開放資源","：相關論文與程式碼已在 GitHub 開源，標誌著大語言模型從離散向連續遷徙的開端。",{"title":360,"searchDepth":361,"depth":361,"links":362},"",2,[363,364,368,369,370,371],{"id":48,"depth":361,"text":49},{"id":80,"depth":361,"text":81,"children":365},[366],{"id":97,"depth":367,"text":98},3,{"id":172,"depth":361,"text":173},{"id":242,"depth":361,"text":243},{"id":281,"depth":361,"text":282},{"id":311,"depth":361,"text":312},"2026-08-06","傳統大模型逐字（Token-by-token）生成的效率已觸及物理天花板。CALM 將離散 Token 轉換為連續向量（Continuous Vectors），將生成步數縮減至原來的 1/K，為大語言模型的擴充開闢了全新的效率路徑。","md","/images/2510.27688v1.jpg",{},true,"/blog/27688v1",{"title":5,"description":373},{"loc":378},"blog/2510.27688v1",null,"CHEtd_5M7ONL8YpAwa-5BpXcqRIZW6zr6yJea5j2FGs",1785987210639]