[{"data":1,"prerenderedAt":1052},["ShallowReactive",2],{"article-\u002Farticles\u002Fkafka-topic-partition-consumer-group":3},{"id":4,"title":5,"body":6,"date":1038,"description":504,"excerpt":1039,"extension":1040,"meta":1041,"navigation":1042,"path":1043,"seo":1044,"stem":1045,"tags":1046,"__hash__":1051},"articles\u002Farticles\u002Fkafka-topic-partition-consumer-group.md","Kafka 完整入門：Broker、Topic、Partition、Replica、Consumer Group 一次搞懂",{"type":7,"value":8,"toc":1015},"minimark",[9,14,23,44,47,50,173,184,188,195,219,222,227,230,246,249,253,263,277,284,333,340,344,350,360,387,394,408,412,415,421,481,497,507,514,518,531,534,537,563,567,574,613,616,622,630,635,638,645,648,653,660,666,670,677,688,702,708,711,733,743,749,815,819,824,839,842,860,870,874,877,905,909,912,954,957,961,968,999,1002,1011],[10,11,13],"h2",{"id":12},"kafka-是一份分散式日誌","Kafka 是一份分散式日誌",[15,16,17,18,22],"p",{},"理解 Kafka 最好的方式，是把它想成一份",[19,20,21],"strong",{},"只能往後附加（Append-only）的日誌檔","：",[24,25,26,34,41],"ul",{},[27,28,29,30,33],"li",{},"Producer 寫入的訊息，一律",[19,31,32],{},"附加在尾端","，寫入後不能修改",[27,35,36,37,40],{},"訊息被讀取後",[19,38,39],{},"不會刪除","，而是依保留期限（例如 7 天）或容量上限才清除",[27,42,43],{},"每個讀取者自己記住「我讀到第幾筆」，想從頭重讀、從昨天重讀都可以",[15,45,46],{},"這和「訊息被取走就消失」的傳統佇列有本質上的不同，後面所有的設計——Partition、Offset、Consumer Group——都是從這個模型延伸出來的。",[10,48,49],{"id":49},"先認識所有元件",[51,52,53,66],"table",{},[54,55,56],"thead",{},[57,58,59,63],"tr",{},[60,61,62],"th",{},"元件",[60,64,65],{},"是什麼",[67,68,69,80,90,100,119,129,139,149,159],"tbody",{},[57,70,71,77],{},[72,73,74],"td",{},[19,75,76],{},"Broker",[72,78,79],{},"一台 Kafka 伺服器，負責儲存 Partition 的資料、處理讀寫請求",[57,81,82,87],{},[72,83,84],{},[19,85,86],{},"Cluster",[72,88,89],{},"多台 Broker 組成的叢集，資料與負載分散在各台 Broker 上",[57,91,92,97],{},[72,93,94],{},[19,95,96],{},"Controller",[72,98,99],{},"叢集的管理者，負責記錄 Topic 的設定、決定每個 Partition 由誰當 Leader、處理 Broker 上下線",[57,101,102,107],{},[72,103,104],{},[19,105,106],{},"Topic",[72,108,109,110,114,115,118],{},"一類訊息的名稱，例如 ",[111,112,113],"code",{},"orders","、",[111,116,117],{},"app-logs","，是邏輯上的分類",[57,120,121,126],{},[72,122,123],{},[19,124,125],{},"Partition",[72,127,128],{},"Topic 被切成的多份日誌，是平行處理與順序保證的單位",[57,130,131,136],{},[72,132,133],{},[19,134,135],{},"Replica",[72,137,138],{},"每個 Partition 在不同 Broker 上的副本，分成一個 Leader 與多個 Follower",[57,140,141,146],{},[72,142,143],{},[19,144,145],{},"Producer",[72,147,148],{},"寫入訊息的程式",[57,150,151,156],{},[72,152,153],{},[19,154,155],{},"Consumer",[72,157,158],{},"讀取訊息的程式",[57,160,161,166],{},[72,162,163],{},[19,164,165],{},"Consumer Group",[72,167,168,169,172],{},"一群共同分工消費的 Consumer，用 ",[111,170,171],{},"group.id"," 識別",[15,174,175,176,179,180,183],{},"早期的 Kafka 需要另外架一組 ZooKeeper 來存放叢集的中繼資料、選出 Controller。從 2.8 開始可以改用 ",[19,177,178],{},"KRaft"," 模式，由 Kafka 自己內建的 Raft 共識機制管理；今年三月發布的 ",[19,181,182],{},"Kafka 4.0 已經完全移除 ZooKeeper","，新的叢集直接用 KRaft 就好，少維護一整組服務。",[10,185,187],{"id":186},"topic-與-partition","Topic 與 Partition",[15,189,190],{},[191,192],"img",{"alt":193,"src":194},"Kafka Topic 與 Partition 示意圖：Producer 以 hash(key) % 3 決定 Partition，key 為帳號 A001 的訊息寫入 Partition 1 的尾端；Topic orders 有三個 Partition，每個 Partition 的 offset 各自從 0 開始編號，新訊息只會附加在尾端；同一個 Partition 內依 offset 嚴格有序，不同 Partition 之間沒有順序保證，同一個 Key 永遠進同一個 Partition，因此同帳號的事件有序；訊息被讀取後不會刪除，依保留期限或大小清除，不同的 Consumer Group 可以各自從任何 offset 重讀","\u002Fimages\u002Farticles\u002Fkafka-topic-partition.svg",[24,196,197,209],{},[27,198,199,201,202,204,205,208],{},[19,200,106],{}," 實際上被切成多個 ",[19,203,125],{},"，",[19,206,207],{},"每個 Partition 才是一份獨立的日誌","。Partition 分散在叢集中不同的 Broker 上，讀寫的負載也就分散開來",[27,210,211,214,215,218],{},[19,212,213],{},"Offset"," 是訊息在 Partition 中的位置編號，每個 Partition ",[19,216,217],{},"各自從 0 開始","遞增",[15,220,221],{},"Partition 是 Kafka 能夠水平擴展的根本：一個 Topic 有 12 個 Partition，就可以分散在多台 Broker 上同時寫入、同時讀取。",[223,224,226],"h3",{"id":225},"順序只在-partition-內成立","順序只在 Partition 內成立",[15,228,229],{},"這是使用 Kafka 最重要的一件事：",[24,231,232,241],{},[27,233,234,237,238],{},[19,235,236],{},"同一個 Partition 內","，訊息依 Offset ",[19,239,240],{},"嚴格有序",[27,242,243],{},[19,244,245],{},"不同 Partition 之間沒有順序保證",[15,247,248],{},"如果 Topic 有 3 個 Partition，先寫入的訊息 A 在 P0、後寫入的訊息 B 在 P1，消費端完全有可能先處理 B 再處理 A。",[223,250,252],{"id":251},"key-決定訊息進哪個-partition","Key 決定訊息進哪個 Partition",[15,254,255,256,259,260,22],{},"Producer 寫入時可以帶一個 ",[19,257,258],{},"Key","，預設的分區策略是 ",[111,261,262],{},"hash(key) % Partition 數量",[24,264,265,271],{},[27,266,267,270],{},[19,268,269],{},"相同 Key 的訊息，一定進同一個 Partition","，因此彼此之間是有序的",[27,272,273,276],{},[19,274,275],{},"沒有 Key"," 的訊息會被分散到各個 Partition（新版本的預設是 Sticky Partitioner，先把一批訊息送到同一個 Partition 再換下一個），吞吐量最好，但沒有任何順序保證",[15,278,279,280,283],{},"所以 Key 的選擇，就是在決定「",[19,281,282],{},"哪些訊息之間需要有序","」：",[51,285,286,298],{},[54,287,288],{},[57,289,290,292,295],{},[60,291,258],{},[60,293,294],{},"保證",[60,296,297],{},"適合",[67,299,300,311,322],{},[57,301,302,305,308],{},[72,303,304],{},"帳號 ID",[72,306,307],{},"同一個帳號的事件有序",[72,309,310],{},"委託、成交、帳務異動",[57,312,313,316,319],{},[72,314,315],{},"訂單 ID",[72,317,318],{},"同一筆訂單的狀態變化有序",[72,320,321],{},"訂單建立 → 付款 → 出貨",[57,323,324,327,330],{},[72,325,326],{},"不設 Key",[72,328,329],{},"不保證順序，負載最平均",[72,331,332],{},"日誌、監控指標、點擊紀錄",[15,334,335,336,339],{},"Key 也要注意",[19,337,338],{},"分布是否平均","：如果用「商品類別」當 Key，而某個類別佔了八成的訊息，就會有一個 Partition 特別忙（Hot Partition），其他 Partition 閒置，整個 Topic 的吞吐量被那一個 Partition 卡住。",[10,341,343],{"id":342},"replicabroker-掛掉資料也不會掉","Replica：Broker 掛掉資料也不會掉",[15,345,346],{},[191,347],{"alt":348,"src":349},"Kafka 複本示意圖：三台 Broker，orders 的 P0 Leader 在 Broker 1、Follower 在 Broker 2 與 3；P1 Leader 在 Broker 2；P2 Leader 在 Broker 3，Leader 平均分散；Producer 與 Consumer 只和 Leader 讀寫，Follower 持續向 Leader 複製；與 Leader 保持同步的副本集合稱為 ISR；設定 replication.factor=3、min.insync.replicas=2、acks=all 時，訊息要寫進至少兩個同步副本才算成功，可以容忍一台 Broker 故障；Leader 所在的 Broker 掛掉時，Controller 從 ISR 中選出新的 Leader","\u002Fimages\u002Farticles\u002Fkafka-replication.svg",[15,351,352,353,356,357,22],{},"每個 Partition 可以有多個副本，數量由 ",[111,354,355],{},"replication.factor"," 決定，正式環境通常設 ",[19,358,359],{},"3",[24,361,362,371,377],{},[27,363,364,367,368],{},[19,365,366],{},"Leader","：每個 Partition 只有一個 Leader，",[19,369,370],{},"所有讀寫都經過 Leader",[27,372,373,376],{},[19,374,375],{},"Follower","：持續從 Leader 複製資料，平常不對外服務，Leader 掛掉時接手",[27,378,379,382,383,386],{},[19,380,381],{},"ISR（In-Sync Replicas）","：跟 Leader 保持同步、沒有落後太多的副本集合。Follower 太久沒跟上（",[111,384,385],{},"replica.lag.time.max.ms","），就會被踢出 ISR",[15,388,389,390,393],{},"Leader 所在的 Broker 掛掉時，Controller 會從 ",[19,391,392],{},"ISR"," 中選一個 Follower 成為新的 Leader，Producer 和 Consumer 自動改連新的 Leader。每個 Partition 的 Leader 會平均分散在各台 Broker 上，所以每台 Broker 都在分擔讀寫。",[15,395,396,399,400,403,404,407],{},[111,397,398],{},"unclean.leader.election.enable"," 預設是 ",[111,401,402],{},"false","，意思是",[19,405,406],{},"寧可這個 Partition 暫時不能用，也不讓一個落後的副本當 Leader","，因為落後的副本當上 Leader，就代表沒同步到的那些訊息永遠遺失了。",[10,409,411],{"id":410},"producer寫入怎樣才算成功","Producer：寫入怎樣才算成功",[223,413,414],{"id":414},"acks",[15,416,417,418,420],{},"Producer 的 ",[111,419,414],{}," 決定「要多少個副本收到，才算寫入成功」：",[51,422,423,435],{},[54,424,425],{},[57,426,427,429,432],{},[60,428,414],{},[60,430,431],{},"意思",[60,433,434],{},"風險",[67,436,437,450,463],{},[57,438,439,444,447],{},[72,440,441],{},[111,442,443],{},"0",[72,445,446],{},"送出去就算成功，不等回應",[72,448,449],{},"Broker 沒收到也不知道",[57,451,452,457,460],{},[72,453,454],{},[111,455,456],{},"1",[72,458,459],{},"Leader 寫入就算成功",[72,461,462],{},"Leader 寫完、還沒同步給 Follower 就掛了，訊息遺失",[57,464,465,475,478],{},[72,466,467,470,471,474],{},[111,468,469],{},"all","（",[111,472,473],{},"-1","）",[72,476,477],{},"ISR 中所有副本都寫入才算成功",[72,479,480],{},"最安全，延遲稍高",[15,482,483,486,487,492,493,496],{},[111,484,485],{},"acks=all"," 要搭配 Topic 的 ",[19,488,489],{},[111,490,491],{},"min.insync.replicas"," 才有意義。如果 ISR 只剩 Leader 自己，「所有 ISR 都寫入」其實等於 ",[111,494,495],{},"acks=1","。常見的組合是：",[498,499,505],"pre",{"className":500,"code":502,"language":503,"meta":504},[501],"language-text","replication.factor  = 3\nmin.insync.replicas = 2\nacks                = all\n","text","",[111,506,502],{"__ignoreMap":504},[15,508,509,510,513],{},"意思是每則訊息至少要寫進兩個副本才算成功，",[19,511,512],{},"可以容忍一台 Broker 故障","；如果同時掛了兩台，ISR 只剩一個，Producer 會收到錯誤而不是默默把資料寫進一個隨時可能消失的副本。",[223,515,517],{"id":516},"冪等-producer","冪等 Producer",[15,519,520,521,526,527,530],{},"網路抖動時，Producer 可能送出了訊息、但沒收到 Broker 的確認，於是重送，造成同一則訊息寫入兩次。開啟 ",[19,522,523],{},[111,524,525],{},"enable.idempotence=true"," 後，Broker 會用 Producer ID 加序號辨識重送的訊息並丟掉重複的，同時保證重試不會打亂同一個 Partition 內的順序。Java Client 從 Kafka 3.0 開始預設開啟；但 .NET 的 Confluent.Kafka 底層是 librdkafka，預設仍是關閉，要自己設定 ",[111,528,529],{},"EnableIdempotence = true","。",[223,532,533],{"id":533},"批次與壓縮",[15,535,536],{},"Producer 不會每則訊息都送一次請求，而是累積成批次再送：",[24,538,539,545,551],{},[27,540,541,544],{},[111,542,543],{},"linger.ms","：最多等多久湊一批，調大一點（例如 5～20ms）可以明顯提升吞吐量",[27,546,547,550],{},[111,548,549],{},"batch.size","：一批最大多少 Byte",[27,552,553,22,556,114,559,562],{},[111,554,555],{},"compression.type",[111,557,558],{},"lz4",[111,560,561],{},"zstd"," 等壓縮，對日誌這類重複性高的文字效果很好",[10,564,566],{"id":565},"保留與壓縮訊息什麼時候被清掉","保留與壓縮：訊息什麼時候被清掉",[15,568,569,570,573],{},"Topic 的 ",[111,571,572],{},"cleanup.policy"," 有兩種：",[24,575,576,597],{},[27,577,578,584,585,588,589,592,593,596],{},[19,579,580,583],{},[111,581,582],{},"delete","（預設）","：依 ",[111,586,587],{},"retention.ms","（例如 7 天）或 ",[111,590,591],{},"retention.bytes"," 清除舊的資料，",[19,594,595],{},"不管有沒有被讀過","。所以 Consumer 停機超過保留期限，沒讀到的資料就真的沒了",[27,598,599,604,605,608,609,612],{},[19,600,601],{},[111,602,603],{},"compact","：同一個 Key ",[19,606,607],{},"只保留最新的一筆","。適合存放「狀態」，例如每個帳號最新的設定，新的服務上線時從頭讀一遍，就能拿到每個 Key 的最新值。發一則 Value 為 ",[111,610,611],{},"null"," 的訊息（Tombstone）可以刪除這個 Key",[10,614,165],{"id":615},"consumer-group",[15,617,618],{},[191,619],{"alt":620,"src":621},"Kafka Consumer Group 示意圖：Topic app-logs 有三個 Partition；Consumer Group A（log-indexer，寫入搜尋引擎）有四個 Consumer，Consumer 1、2、3 各分到 P0、P1、P2，第四個 Consumer 因為 Partition 不夠而閒置，組內分工時一個 Partition 只給組內一個 Consumer；Consumer Group B（log-archiver，封存到物件儲存）只有一個 Consumer，同時讀取 P0、P1、P2，組與組之間互不影響，每個 Group 都讀到完整的資料並各自記錄 offset","\u002Fimages\u002Farticles\u002Fkafka-consumer-groups.svg",[15,623,624,626,627,629],{},[19,625,165],{}," 是一群共同消費同一個 Topic 的 Consumer，用 ",[111,628,171],{}," 識別。它同時解決了兩件事：",[15,631,632],{},[19,633,634],{},"1. 組內分工：一個 Partition 只會分給組內的一個 Consumer",[15,636,637],{},"Group A 有 3 個 Consumer、Topic 有 3 個 Partition，剛好一人一個，三個 Consumer 平行處理。",[15,639,640,641,644],{},"這也代表：",[19,642,643],{},"Partition 的數量，就是一個 Consumer Group 的平行度上限","。Group A 開到第 4 個 Consumer 時，已經沒有 Partition 可以分給它，它只會閒置，什麼都不做，只有在其他 Consumer 離開、觸發重新分配時才可能分到工作。想要再提高消費速度，必須先增加 Partition。",[15,646,647],{},"反過來，Consumer 比 Partition 少時，一個 Consumer 會同時負責多個 Partition，例如 Group B 只有 1 個 Consumer，就自己讀全部 3 個 Partition。",[15,649,650],{},[19,651,652],{},"2. 組與組之間互不影響：每個 Group 都讀到完整的資料",[15,654,655,656,659],{},"Group A 把日誌寫進搜尋引擎、Group B 把日誌封存到物件儲存，兩者都讀同一個 Topic，",[19,657,658],{},"各自讀到全部的訊息","，各自記錄自己讀到的 Offset。Group B 處理得比較慢，完全不會影響 Group A。",[15,661,662,663],{},"一句話記：",[19,664,665],{},"同一個 Group 內是分工，不同 Group 之間是各自完整讀一份。",[223,667,669],{"id":668},"offset-由-consumer-group-自己記錄","Offset 由 Consumer Group 自己記錄",[15,671,672,673,676],{},"每個 Consumer Group 會把自己在每個 Partition 讀到的位置，提交（Commit）回 Kafka 內部的 ",[111,674,675],{},"__consumer_offsets"," Topic。Consumer 重啟或換人接手時，就從上次提交的位置繼續讀。",[15,678,679,680,683,684,687],{},"一個",[19,681,682],{},"全新的"," Group（還沒有任何提交紀錄）要從哪裡開始讀，由 ",[111,685,686],{},"auto.offset.reset"," 決定：",[24,689,690,696],{},[27,691,692,695],{},[111,693,694],{},"latest","（預設）：從現在開始，之前的訊息都不讀",[27,697,698,701],{},[111,699,700],{},"earliest","：從保留範圍內最舊的訊息開始讀",[15,703,704,705,707],{},"新上線的服務如果需要補齊歷史資料，要記得設成 ",[111,706,700],{},"，不然會以為「怎麼少了之前的資料」。",[15,709,710],{},"提交的時機決定了投遞語意：",[24,712,713,723],{},[27,714,715,718,719,722],{},[19,716,717],{},"先提交、再處理","：處理到一半當掉，重啟後會從下一筆開始，這一筆",[19,720,721],{},"遺失","（At-most-once）",[27,724,725,728,729,732],{},[19,726,727],{},"先處理、再提交","：處理完、提交前當掉，重啟後會",[19,730,731],{},"重複處理","這一筆（At-least-once）",[15,734,735,738,739,742],{},[111,736,737],{},"enable.auto.commit=true","（預設）會在背景每 5 秒自動提交「已經拉回來的位置」，不管你處理完了沒有，所以兩種情況都可能發生。需要可靠處理時，建議關掉自動提交，處理完再手動提交。大多數系統選擇 At-least-once，並讓處理邏輯",[19,740,741],{},"冪等","，重複處理也不會出錯。",[15,744,745,746,22],{},"因為 Offset 可以自由調整，Kafka 還能做到傳統佇列做不到的事：發現某段時間的資料處理錯了，修好程式後",[19,747,748],{},"把 Offset 倒回去重新消費",[498,750,754],{"className":751,"code":752,"language":753,"meta":504,"style":504},"language-bash shiki shiki-themes github-dark","# 把 log-indexer 這個 Group 倒回到昨天早上 9 點（Group 要先停止所有 Consumer）\nkafka-consumer-groups.sh --bootstrap-server kafka:9092 \\\n  --group log-indexer --topic app-logs \\\n  --reset-offsets --to-datetime 2025-05-12T09:00:00.000 --execute\n","bash",[111,755,756,765,783,800],{"__ignoreMap":504},[757,758,761],"span",{"class":759,"line":760},"line",1,[757,762,764],{"class":763},"sAwPA","# 把 log-indexer 這個 Group 倒回到昨天早上 9 點（Group 要先停止所有 Consumer）\n",[757,766,768,772,776,780],{"class":759,"line":767},2,[757,769,771],{"class":770},"svObZ","kafka-consumer-groups.sh",[757,773,775],{"class":774},"sDLfK"," --bootstrap-server",[757,777,779],{"class":778},"sU2Wk"," kafka:9092",[757,781,782],{"class":774}," \\\n",[757,784,786,789,792,795,798],{"class":759,"line":785},3,[757,787,788],{"class":774},"  --group",[757,790,791],{"class":778}," log-indexer",[757,793,794],{"class":774}," --topic",[757,796,797],{"class":778}," app-logs",[757,799,782],{"class":774},[757,801,803,806,809,812],{"class":759,"line":802},4,[757,804,805],{"class":774},"  --reset-offsets",[757,807,808],{"class":774}," --to-datetime",[757,810,811],{"class":778}," 2025-05-12T09:00:00.000",[757,813,814],{"class":774}," --execute\n",[223,816,818],{"id":817},"rebalance","Rebalance",[15,820,821,822,530],{},"Consumer Group 的成員變動時——新增 Consumer、某個 Consumer 當掉或太久沒有回應、Topic 的 Partition 數量改變——Kafka 會重新分配 Partition，這個過程叫 ",[19,823,818],{},[15,825,826,827,830,831,834,835,838],{},"Rebalance 期間，相關的 Partition 會暫停消費。傳統的做法（Eager）是",[19,828,829],{},"所有 Consumer 先放掉全部 Partition，再重新分配","，整個 Group 停下來一次；改用 Cooperative Sticky 的漸進式 Rebalance（librdkafka／.NET 設定 ",[111,832,833],{},"partition.assignment.strategy=cooperative-sticky","，Java 是 ",[111,836,837],{},"CooperativeStickyAssignor","），只會搬動需要移動的 Partition，其他的照常消費。",[15,840,841],{},"判斷 Consumer 是否還活著有兩個逾時：",[24,843,844,850],{},[27,845,846,849],{},[111,847,848],{},"session.timeout.ms","（預設 45 秒）：多久沒收到心跳就判定 Consumer 掛了",[27,851,852,855,856,859],{},[111,853,854],{},"max.poll.interval.ms","（預設 5 分鐘）：兩次 ",[111,857,858],{},"poll()"," 之間最多隔多久",[15,861,862,863,865,866,869],{},"常見的問題是處理單批訊息的時間太長，超過 ",[111,864,854],{},"，Kafka 以為這個 Consumer 卡住而把它踢出 Group、觸發 Rebalance，接著另一個 Consumer 接手、重複處理，一直循環。處理時間長的工作，要調小每次拉取的筆數（",[111,867,868],{},"max.poll.records","）或調大這個逾時設定。",[10,871,873],{"id":872},"partition-要開幾個","Partition 要開幾個？",[15,875,876],{},"Partition 數量是設計 Topic 時最重要的決定之一：",[24,878,879,885,899],{},[27,880,881,884],{},[19,882,883],{},"由消費端反推","：預估尖峰時每秒要處理多少訊息、單一 Consumer 每秒能處理多少，相除就是需要的平行度，Partition 數量至少要這麼多，再預留成長空間",[27,886,887,890,891,898],{},[19,888,889],{},"可以增加、不能減少","：Partition 可以事後增加，但",[19,892,893,894,897],{},"增加後 ",[111,895,896],{},"hash(key) % N"," 的結果會改變","，同一個 Key 之後會進到不同的 Partition，增加的那一刻前後的順序就無法保證。對順序敏感的 Topic，一開始就要開足",[27,900,901,904],{},[19,902,903],{},"不是越多越好","：每個 Partition 都有檔案控制代碼、複本同步、Rebalance 的成本，數量過多會拉長故障恢復與 Rebalance 的時間",[10,906,908],{"id":907},"使用情境一集中式日誌收集","使用情境一：集中式日誌收集",[15,910,911],{},"跨機房的應用程式日誌全部送進 Kafka，再由不同的系統各自消費：",[24,913,914,920,929,938,946],{},[27,915,916,917,919],{},"Topic ",[111,918,117],{}," 不設 Key，讓訊息平均分散在所有 Partition，追求最大吞吐量——日誌之間不需要嚴格順序",[27,921,922,923,925,926,928],{},"Producer 開啟 ",[111,924,558],{}," 壓縮與較大的 ",[111,927,543],{},"，日誌的文字重複性高，壓縮後網路與磁碟用量都能省下一大截",[27,930,931,937],{},[19,932,933,934],{},"Group ",[111,935,936],{},"log-indexer","：寫入搜尋引擎供查詢，Consumer 數量隨 Partition 數量水平擴展",[27,939,940,945],{},[19,941,933,942],{},[111,943,944],{},"log-archiver","：封存到物件儲存做長期保存",[27,947,948,949,951,952],{},"搜尋引擎維護或故障時，",[111,950,936],{}," 停下來，日誌仍然安全地留在 Kafka 中；恢復後從上次的 Offset 繼續追上，不會遺失——前提是停機時間不能超過 ",[111,953,587],{},[15,955,956],{},"日誌量成長時，增加 Partition 與 Consumer 就能擴充處理量；Kafka 本身也成為應用程式與搜尋引擎之間的緩衝，尖峰流量不會直接打垮後端。",[10,958,960],{"id":959},"使用情境二需要順序的交易事件","使用情境二：需要順序的交易事件",[15,962,963,964,967],{},"委託、成交、帳務異動這類事件，",[19,965,966],{},"同一個帳號的事件順序不能錯","：先成交再取消，和先取消再成交，結果完全不同。",[24,969,970,977,980,993,996],{},[27,971,972,973,976],{},"以",[19,974,975],{},"帳號 ID 作為 Key","，同一個帳號的所有事件都進同一個 Partition，依序被同一個 Consumer 處理",[27,978,979],{},"不同帳號之間不需要順序，分散在不同 Partition 平行處理",[27,981,982,983,985,986,114,989,992],{},"Producer 使用 ",[111,984,485],{}," 加冪等，Topic 設定 ",[111,987,988],{},"replication.factor=3",[111,990,991],{},"min.insync.replicas=2","，一台 Broker 故障也不掉資料",[27,994,995],{},"Consumer 關閉自動提交，採用「先處理、再提交 Offset」，並以事件 ID 做冪等檢查，重複投遞也不會重複入帳",[27,997,998],{},"一開始就依預估量開足 Partition，避免日後擴充造成同一帳號的事件跨 Partition",[10,1000,1001],{"id":1001},"一句話總結",[15,1003,1004,1005,1007,1008,1010],{},"Kafka 是一份可重複讀取的分散式日誌：Topic 切成多個 Partition 分散在各台 Broker，每個 Partition 再複製成 Leader 與 Follower，搭配 ",[111,1006,485],{}," 與 ",[111,1009,491],{}," 撐過 Broker 故障；順序只在 Partition 內成立，所以用 Key 決定「哪些訊息要有序」；Consumer Group 內一個 Partition 只給一個 Consumer，因此 Partition 數量就是平行度上限，不同 Group 各自讀到完整的資料並各自記錄 Offset，讓同一份資料能被多個系統獨立、可重播地消費。",[1012,1013,1014],"style",{},"html pre.shiki code .sAwPA, html code.shiki .sAwPA{--shiki-default:#6A737D}html pre.shiki code .svObZ, html code.shiki .svObZ{--shiki-default:#B392F0}html pre.shiki code .sDLfK, html code.shiki .sDLfK{--shiki-default:#79B8FF}html pre.shiki code .sU2Wk, html code.shiki .sU2Wk{--shiki-default:#9ECBFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}",{"title":504,"searchDepth":767,"depth":767,"links":1016},[1017,1018,1019,1023,1024,1029,1030,1034,1035,1036,1037],{"id":12,"depth":767,"text":13},{"id":49,"depth":767,"text":49},{"id":186,"depth":767,"text":187,"children":1020},[1021,1022],{"id":225,"depth":785,"text":226},{"id":251,"depth":785,"text":252},{"id":342,"depth":767,"text":343},{"id":410,"depth":767,"text":411,"children":1025},[1026,1027,1028],{"id":414,"depth":785,"text":414},{"id":516,"depth":785,"text":517},{"id":533,"depth":785,"text":533},{"id":565,"depth":767,"text":566},{"id":615,"depth":767,"text":165,"children":1031},[1032,1033],{"id":668,"depth":785,"text":669},{"id":817,"depth":785,"text":818},{"id":872,"depth":767,"text":873},{"id":907,"depth":767,"text":908},{"id":959,"depth":767,"text":960},{"id":1001,"depth":767,"text":1001},"2025-05-13",null,"md",{},true,"\u002Farticles\u002Fkafka-topic-partition-consumer-group",{"title":5,"description":504},"articles\u002Fkafka-topic-partition-consumer-group",[1047,1048,1049,1050],"Kafka","事件串流","分散式系統","系統設計","0K0d4ZhyVYdOR_ZG8z3r3zS0vwdA_5ROeYwDyY2lsFk",1792959150223]