[{"data":1,"prerenderedAt":786},["ShallowReactive",2],{"article-\u002Farticles\u002Fsignalr-streaming-gpt":3},{"id":4,"title":5,"body":6,"date":774,"description":161,"excerpt":775,"extension":776,"meta":777,"navigation":216,"path":778,"seo":779,"stem":780,"tags":781,"__hash__":785},"articles\u002Farticles\u002Fsignalr-streaming-gpt.md","用 SignalR 串流做出 ChatGPT 的「逐字出現」：IAsyncEnumerable、取消與多實例部署",{"type":7,"value":8,"toc":764},"minimark",[9,14,23,30,53,60,64,67,129,135,139,155,460,476,479,500,504,515,619,630,634,648,656,663,666,669,672,677,684,696,701,708,730,734,737,748,751,760],[10,11,13],"h2",{"id":12},"問題等待時間不變但感受差很多","問題：等待時間不變，但感受差很多",[15,16,17,18,22],"p",{},"大型語言模型是",[19,20,21],"strong",{},"一個 Token 一個 Token 生成","回答的。一段 800 字的回答，模型可能要花 15～20 秒才全部生成完。",[15,24,25],{},[26,27],"img",{"alt":28,"src":29},"一般 HTTP 與串流的比較：一般 HTTP 要等全部生成完才一次回傳，第一個字大約在 20 秒才出現；串流則是產生一段就推一段，第一個字大約 1 秒就出現。下方是串流的管線：OpenAI API 以 stream 模式回傳，SignalR Hub 以 IAsyncEnumerable 逐段 yield return，透過 WebSocket 推到瀏覽器的 connection.stream","\u002Fimages\u002Farticles\u002Fsignalr-streaming.svg",[31,32,33,43],"ul",{},[34,35,36,39,40],"li",{},[19,37,38],{},"一般 HTTP 請求","：後端等 OpenAI 全部生成完、拿到完整回答後才回傳。總時間 20 秒，",[19,41,42],{},"第一個字也在 20 秒才出現",[34,44,45,48,49,52],{},[19,46,47],{},"串流","：OpenAI 每生成一小段就送出一小段，後端立刻轉給瀏覽器。總時間一樣是 20 秒，但",[19,50,51],{},"第一個字在 1 秒內就出現","，使用者看著文字一段一段長出來，感覺系統一直在回應",[15,54,55,56,59],{},"對使用者體驗來說，關鍵指標不是總時間，而是",[19,57,58],{},"第一個字出現的時間（Time To First Token）","。",[10,61,63],{"id":62},"為什麼選-signalr","為什麼選 SignalR",[15,65,66],{},"要讓伺服器「連續、主動」把資料推給瀏覽器，常見有三種做法：",[68,69,70,86],"table",{},[71,72,73],"thead",{},[74,75,76,80,83],"tr",{},[77,78,79],"th",{},"做法",[77,81,82],{},"特性",[77,84,85],{},"適合",[87,88,89,103,116],"tbody",{},[74,90,91,97,100],{},[92,93,94],"td",{},[19,95,96],{},"Server-Sent Events（SSE）",[92,98,99],{},"單向：伺服器 → 瀏覽器；基於一般 HTTP",[92,101,102],{},"只需要伺服器推送，例如通知、單一問答的串流",[74,104,105,110,113],{},[92,106,107],{},[19,108,109],{},"WebSocket",[92,111,112],{},"雙向、全雙工，連線建立後雙方隨時可以傳送",[92,114,115],{},"需要雙向即時互動，例如聊天、協作",[74,117,118,123,126],{},[92,119,120],{},[19,121,122],{},"SignalR",[92,124,125],{},"ASP.NET Core 的即時通訊框架，優先使用 WebSocket，不支援時自動降級為 SSE 或 Long Polling",[92,127,128],{},".NET 後端、需要雙向通訊，不想自己處理連線管理",[15,130,131,132,134],{},"聊天室除了串流回覆，還有「使用者中途按停止」「其他分頁同步更新聊天室列表」這些雙向需求。後端是 ASP.NET Core 的情況下，",[19,133,122],{}," 幫忙處理了連線建立、重新連線、訊息序列化、降級傳輸等細節，而且直接支援串流。",[10,136,138],{"id":137},"後端hub-方法回傳-iasyncenumerable","後端：Hub 方法回傳 IAsyncEnumerable",[15,140,141,142,146,147,150,151,154],{},"SignalR 的 Hub 方法只要回傳 ",[143,144,145],"code",{},"IAsyncEnumerable\u003CT>","，就自動變成",[19,148,149],{},"串流方法","：每 ",[143,152,153],{},"yield return"," 一個值，就推一個值給前端。",[156,157,162],"pre",{"className":158,"code":159,"language":160,"meta":161,"style":161},"language-csharp shiki shiki-themes github-dark","public class ChatHub : Hub\n{\n    private readonly IChatService _chatService;\n\n    public ChatHub(IChatService chatService) => _chatService = chatService;\n\n    public async IAsyncEnumerable\u003Cstring> SendMessage(\n        Guid chatRoomId,\n        string message,\n        [EnumeratorCancellation] CancellationToken cancellationToken)\n    {\n        var answer = new StringBuilder();\n\n        await foreach (var chunk in _chatService.StreamCompletionAsync(chatRoomId, message, cancellationToken))\n        {\n            answer.Append(chunk);\n            yield return chunk;\n        }\n\n        \u002F\u002F 串流結束後才寫入資料庫，而不是每一段都寫一次\n        await _chatService.SaveAnswerAsync(chatRoomId, answer.ToString(), CancellationToken.None);\n    }\n}\n","csharp","",[143,163,164,187,193,211,218,250,255,281,293,304,325,331,352,357,387,393,405,411,417,422,429,448,454],{"__ignoreMap":161},[165,166,169,173,176,180,184],"span",{"class":167,"line":168},"line",1,[165,170,172],{"class":171},"snl16","public",[165,174,175],{"class":171}," class",[165,177,179],{"class":178},"svObZ"," ChatHub",[165,181,183],{"class":182},"s95oV"," : ",[165,185,186],{"class":178},"Hub\n",[165,188,190],{"class":167,"line":189},2,[165,191,192],{"class":182},"{\n",[165,194,196,199,202,205,208],{"class":167,"line":195},3,[165,197,198],{"class":171},"    private",[165,200,201],{"class":171}," readonly",[165,203,204],{"class":178}," IChatService",[165,206,207],{"class":178}," _chatService",[165,209,210],{"class":182},";\n",[165,212,214],{"class":167,"line":213},4,[165,215,217],{"emptyLinePlaceholder":216},true,"\n",[165,219,221,224,226,229,232,235,238,241,244,247],{"class":167,"line":220},5,[165,222,223],{"class":171},"    public",[165,225,179],{"class":178},[165,227,228],{"class":182},"(",[165,230,231],{"class":178},"IChatService",[165,233,234],{"class":178}," chatService",[165,236,237],{"class":182},") ",[165,239,240],{"class":171},"=>",[165,242,243],{"class":182}," _chatService ",[165,245,246],{"class":171},"=",[165,248,249],{"class":182}," chatService;\n",[165,251,253],{"class":167,"line":252},6,[165,254,217],{"emptyLinePlaceholder":216},[165,256,258,260,263,266,269,272,275,278],{"class":167,"line":257},7,[165,259,223],{"class":171},[165,261,262],{"class":171}," async",[165,264,265],{"class":178}," IAsyncEnumerable",[165,267,268],{"class":182},"\u003C",[165,270,271],{"class":171},"string",[165,273,274],{"class":182},"> ",[165,276,277],{"class":178},"SendMessage",[165,279,280],{"class":182},"(\n",[165,282,284,287,290],{"class":167,"line":283},8,[165,285,286],{"class":178},"        Guid",[165,288,289],{"class":178}," chatRoomId",[165,291,292],{"class":182},",\n",[165,294,296,299,302],{"class":167,"line":295},9,[165,297,298],{"class":171},"        string",[165,300,301],{"class":178}," message",[165,303,292],{"class":182},[165,305,307,310,313,316,319,322],{"class":167,"line":306},10,[165,308,309],{"class":182},"        [",[165,311,312],{"class":178},"EnumeratorCancellation",[165,314,315],{"class":182},"] ",[165,317,318],{"class":178},"CancellationToken",[165,320,321],{"class":178}," cancellationToken",[165,323,324],{"class":182},")\n",[165,326,328],{"class":167,"line":327},11,[165,329,330],{"class":182},"    {\n",[165,332,334,337,340,343,346,349],{"class":167,"line":333},12,[165,335,336],{"class":171},"        var",[165,338,339],{"class":178}," answer",[165,341,342],{"class":171}," =",[165,344,345],{"class":171}," new",[165,347,348],{"class":178}," StringBuilder",[165,350,351],{"class":182},"();\n",[165,353,355],{"class":167,"line":354},13,[165,356,217],{"emptyLinePlaceholder":216},[165,358,360,363,366,369,372,375,378,381,384],{"class":167,"line":359},14,[165,361,362],{"class":171},"        await",[165,364,365],{"class":171}," foreach",[165,367,368],{"class":182}," (",[165,370,371],{"class":171},"var",[165,373,374],{"class":178}," chunk",[165,376,377],{"class":171}," in",[165,379,380],{"class":182}," _chatService.",[165,382,383],{"class":178},"StreamCompletionAsync",[165,385,386],{"class":182},"(chatRoomId, message, cancellationToken))\n",[165,388,390],{"class":167,"line":389},15,[165,391,392],{"class":182},"        {\n",[165,394,396,399,402],{"class":167,"line":395},16,[165,397,398],{"class":182},"            answer.",[165,400,401],{"class":178},"Append",[165,403,404],{"class":182},"(chunk);\n",[165,406,408],{"class":167,"line":407},17,[165,409,410],{"class":182},"            yield return chunk;\n",[165,412,414],{"class":167,"line":413},18,[165,415,416],{"class":182},"        }\n",[165,418,420],{"class":167,"line":419},19,[165,421,217],{"emptyLinePlaceholder":216},[165,423,425],{"class":167,"line":424},20,[165,426,428],{"class":427},"sAwPA","        \u002F\u002F 串流結束後才寫入資料庫，而不是每一段都寫一次\n",[165,430,432,434,436,439,442,445],{"class":167,"line":431},21,[165,433,362],{"class":171},[165,435,380],{"class":182},[165,437,438],{"class":178},"SaveAnswerAsync",[165,440,441],{"class":182},"(chatRoomId, answer.",[165,443,444],{"class":178},"ToString",[165,446,447],{"class":182},"(), CancellationToken.None);\n",[165,449,451],{"class":167,"line":450},22,[165,452,453],{"class":182},"    }\n",[165,455,457],{"class":167,"line":456},23,[165,458,459],{"class":182},"}\n",[15,461,462,464,465,468,469,472,473,475],{},[143,463,383],{}," 呼叫 OpenAI API 時設定 ",[143,466,467],{},"stream: true","，OpenAI 會以 SSE 格式一段一段回傳，每一段解析出 ",[143,470,471],{},"delta.content"," 後再 ",[143,474,153],{}," 出去。整條管線從 OpenAI 到瀏覽器都是串流，中間沒有任何地方需要「等全部收齊」。",[15,477,478],{},"幾個設計重點：",[31,480,481,487],{},[34,482,483,486],{},[19,484,485],{},"串流中不寫資料庫","：每一段文字只推給前端，等整段回答結束才一次寫入完整訊息。一段回答可能有幾百段，每一段都寫一次資料庫完全沒有必要",[34,488,489,495,496,499],{},[19,490,491,494],{},[143,492,493],{},"[EnumeratorCancellation]"," 一定要加","：這個屬性讓前端取消串流時，",[143,497,498],{},"cancellationToken"," 會被觸發。沒有加的話，前端取消了，後端還會繼續跑完",[10,501,503],{"id":502},"前端connectionstream","前端：connection.stream()",[15,505,506,507,510,511,514],{},"前端用 ",[143,508,509],{},"@microsoft\u002Fsignalr"," 的 ",[143,512,513],{},"stream()"," 呼叫串流方法，每收到一段就接在目前的回答後面：",[156,516,520],{"className":517,"code":518,"language":519,"meta":161,"style":161},"language-javascript shiki shiki-themes github-dark","const connection = new signalR.HubConnectionBuilder()\n  .withUrl(\"\u002Fhubs\u002Fchat\", { accessTokenFactory: () => getAccessToken() })\n  .withAutomaticReconnect()\n  .build();\n\nawait connection.start();\n\nlet answer = \"\";\nconst subscription = connection.stream(\"SendMessage\", chatRoomId, message)\n  .subscribe({\n    next: (chunk) => {\n      answer += chunk;\n      renderMarkdown(answer);     \u002F\u002F 每次重新渲染，表格、程式碼區塊會逐步成形\n    },\n    complete: () => setStatus(\"done\"),\n    error: (err) => setStatus(\"error\", err),\n  });\n\n\u002F\u002F 使用者按下「停止生成」\nstopButton.onclick = () => subscription.dispose();\n","javascript",[143,521,522,527,532,537,542,546,551,555,560,565,570,575,580,585,590,595,600,605,609,614],{"__ignoreMap":161},[165,523,524],{"class":167,"line":168},[165,525,526],{},"const connection = new signalR.HubConnectionBuilder()\n",[165,528,529],{"class":167,"line":189},[165,530,531],{},"  .withUrl(\"\u002Fhubs\u002Fchat\", { accessTokenFactory: () => getAccessToken() })\n",[165,533,534],{"class":167,"line":195},[165,535,536],{},"  .withAutomaticReconnect()\n",[165,538,539],{"class":167,"line":213},[165,540,541],{},"  .build();\n",[165,543,544],{"class":167,"line":220},[165,545,217],{"emptyLinePlaceholder":216},[165,547,548],{"class":167,"line":252},[165,549,550],{},"await connection.start();\n",[165,552,553],{"class":167,"line":257},[165,554,217],{"emptyLinePlaceholder":216},[165,556,557],{"class":167,"line":283},[165,558,559],{},"let answer = \"\";\n",[165,561,562],{"class":167,"line":295},[165,563,564],{},"const subscription = connection.stream(\"SendMessage\", chatRoomId, message)\n",[165,566,567],{"class":167,"line":306},[165,568,569],{},"  .subscribe({\n",[165,571,572],{"class":167,"line":327},[165,573,574],{},"    next: (chunk) => {\n",[165,576,577],{"class":167,"line":333},[165,578,579],{},"      answer += chunk;\n",[165,581,582],{"class":167,"line":354},[165,583,584],{},"      renderMarkdown(answer);     \u002F\u002F 每次重新渲染，表格、程式碼區塊會逐步成形\n",[165,586,587],{"class":167,"line":359},[165,588,589],{},"    },\n",[165,591,592],{"class":167,"line":389},[165,593,594],{},"    complete: () => setStatus(\"done\"),\n",[165,596,597],{"class":167,"line":395},[165,598,599],{},"    error: (err) => setStatus(\"error\", err),\n",[165,601,602],{"class":167,"line":407},[165,603,604],{},"  });\n",[165,606,607],{"class":167,"line":413},[165,608,217],{"emptyLinePlaceholder":216},[165,610,611],{"class":167,"line":419},[165,612,613],{},"\u002F\u002F 使用者按下「停止生成」\n",[165,615,616],{"class":167,"line":424},[165,617,618],{},"stopButton.onclick = () => subscription.dispose();\n",[15,620,621,622,625,626,629],{},"注意 ",[143,623,624],{},"next"," 收到的是",[19,627,628],{},"片段","，可能只是半個 Markdown 語法（例如表格只出現了一半）。每次都把「目前為止的完整文字」重新渲染一次，比試著增量渲染片段簡單得多，也不會出現語法錯亂。",[10,631,633],{"id":632},"使用者按停止取消要一路傳到底","使用者按停止：取消要一路傳到底",[15,635,636,637,640,641,643,644,647],{},"使用者按下「停止生成」時，",[143,638,639],{},"subscription.dispose()"," 會通知伺服器取消串流，Hub 方法的 ",[143,642,498],{}," 被觸發。這個 Token 要",[19,645,646],{},"一路傳下去","：",[156,649,654],{"className":650,"code":652,"language":653,"meta":161},[651],"language-text","subscription.dispose()\n  → Hub 方法的 cancellationToken\n    → StreamCompletionAsync(..., cancellationToken)\n      → HttpClient 呼叫 OpenAI 的請求被中斷\n","text",[143,655,652],{"__ignoreMap":161},[15,657,658,659,662],{},"如果中間任何一層沒有把 Token 傳下去，畫面上雖然停了，後端仍然在背景繼續向 OpenAI 收資料，",[19,660,661],{},"Token 費用照算","。關掉瀏覽器分頁、網路斷線時，SignalR 連線中斷也會觸發同一個取消流程。",[15,664,665],{},"被取消的回答要不要存？比較好的做法是把「已經生成的部分」存下來並標記為中斷，使用者回到聊天室時，還看得到停在哪裡。",[10,667,668],{"id":668},"多台伺服器時要注意的事",[15,670,671],{},"服務只有一台的時候一切都很單純，但一開始做水平擴展，SignalR 有兩個常見的問題：",[15,673,674],{},[19,675,676],{},"1. 連線協商需要黏著（Sticky Session）",[15,678,679,680,683],{},"SignalR 建立連線時會先發一個 ",[143,681,682],{},"negotiate"," 請求，再用拿到的連線 ID 建立正式連線。如果這兩個請求被負載平衡器分到不同的伺服器，連線就會失敗。解法有兩種：",[31,685,686,689],{},[34,687,688],{},"負載平衡器（或 Kubernetes Ingress）開啟 Sticky Session",[34,690,691,692,695],{},"確定所有用戶端都支援 WebSocket 時，前端設定 ",[143,693,694],{},"skipNegotiation: true"," 並指定只用 WebSocket，就不需要黏著",[15,697,698],{},[19,699,700],{},"2. 推播給「其他連線」需要 Backplane",[15,702,703,704,707],{},"串流回覆是在同一條連線上一問一答，不受影響。但如果要做到「使用者在 A 分頁新增了聊天室，B 分頁的列表也同步更新」，而兩個分頁剛好連到不同的伺服器，伺服器 A 就推不到 B 分頁。這時需要 ",[19,705,706],{},"Backplane","（例如 Redis）讓所有伺服器之間轉送訊息：",[156,709,711],{"className":158,"code":710,"language":160,"meta":161,"style":161},"builder.Services.AddSignalR().AddStackExchangeRedis(redisConnectionString);\n",[143,712,713],{"__ignoreMap":161},[165,714,715,718,721,724,727],{"class":167,"line":168},[165,716,717],{"class":182},"builder.Services.",[165,719,720],{"class":178},"AddSignalR",[165,722,723],{"class":182},"().",[165,725,726],{"class":178},"AddStackExchangeRedis",[165,728,729],{"class":182},"(redisConnectionString);\n",[10,731,733],{"id":732},"使用情境企業內部-gpt-平台","使用情境：企業內部 GPT 平台",[15,735,736],{},"在公司內部的 GPT 平台中，聊天室的對話就是用這套方式實作：",[31,738,739,742,745],{},[34,740,741],{},"前端仿照 ChatGPT 的介面，使用者送出訊息後，回答透過 SignalR 串流逐字出現",[34,743,744],{},"後端以串流模式呼叫 GPT-4，每一段都立即推給前端，整段結束後才把完整回答與 Token 用量寫進聊天室",[34,746,747],{},"使用者按下停止或關閉分頁時，取消會一路傳到 OpenAI 的請求，避免在背景繼續消耗 Token",[10,749,750],{"id":750},"一句話總結",[15,752,753,754,756,757,759],{},"使用者感受到的速度取決於第一個字什麼時候出現，而不是總時間；用 SignalR Hub 回傳 ",[143,755,145],{},"，就能把 OpenAI 的串流回覆一段一段推到瀏覽器，記得加上 ",[143,758,493],{}," 讓取消一路傳到底、串流結束後才寫資料庫，多台部署時處理好 Sticky Session 與 Backplane。",[761,762,763],"style",{},"html pre.shiki code .snl16, html code.shiki .snl16{--shiki-default:#F97583}html pre.shiki code .svObZ, html code.shiki .svObZ{--shiki-default:#B392F0}html pre.shiki code .s95oV, html code.shiki .s95oV{--shiki-default:#E1E4E8}html pre.shiki code .sAwPA, html code.shiki .sAwPA{--shiki-default:#6A737D}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}",{"title":161,"searchDepth":189,"depth":189,"links":765},[766,767,768,769,770,771,772,773],{"id":12,"depth":189,"text":13},{"id":62,"depth":189,"text":63},{"id":137,"depth":189,"text":138},{"id":502,"depth":189,"text":503},{"id":632,"depth":189,"text":633},{"id":668,"depth":189,"text":668},{"id":732,"depth":189,"text":733},{"id":750,"depth":189,"text":750},"2023-07-18",null,"md",{},"\u002Farticles\u002Fsignalr-streaming-gpt",{"title":5,"description":161},"articles\u002Fsignalr-streaming-gpt",[122,109,782,783,784],"ASP.NET Core","OpenAI","即時通訊","yf7eqCfz8RkW9OZ4FN_qBkuTGX_Ra-if6e0Lq7jtpps",1792959150522]