[{"data":1,"prerenderedAt":642},["ShallowReactive",2],{"article-\u002Farticles\u002Fincremental-sync-change-column":3},{"id":4,"title":5,"body":6,"date":627,"description":118,"excerpt":628,"extension":629,"meta":630,"navigation":631,"path":632,"seo":633,"stem":634,"tags":635,"__hash__":641},"articles\u002Farticles\u002Fincremental-sync-change-column.md","從 MSSQL 增量同步到 Elasticsearch：用異動欄位與 Checkpoint 只同步「變動的資料」",{"type":7,"value":8,"toc":612},"minimark",[9,13,21,28,44,51,55,62,65,82,85,112,173,179,183,188,195,229,233,244,271,305,311,315,328,374,385,440,444,451,454,478,481,485,491,506,513,519,522,525,545,548,552,555,569,572,576,579,593,596,599,608],[10,11,12],"h2",{"id":12},"整批重建的極限",[14,15,16,17],"p",{},"讀取改由 Elasticsearch 提供之後，接下來的問題就是：",[18,19,20],"strong",{},"MSSQL 的資料改了，索引要怎麼跟上？",[14,22,23,24,27],{},"最直覺的方式是定期",[18,25,26],{},"整批重建","：每隔一段時間把所有資料讀出來，全部重新寫進 Elasticsearch。它的優點是簡單、一定正確，但：",[29,30,31,35,38],"ul",{},[32,33,34],"li",{},"物件資料有幾十萬筆，一次重建要幾十分鐘，期間資料庫與 Elasticsearch 的負載都很高",[32,36,37],{},"重建的間隔就是資料的延遲：一小時重建一次，經紀人改了價格，前台最久要一小時後才看得到",[32,39,40,41],{},"每次重建處理的資料，",[18,42,43],{},"99% 根本沒有變動",[14,45,46,47,50],{},"整批重建仍然需要（第一次建立索引、Mapping 調整時），但日常的更新應該只處理「有變動的資料」，也就是",[18,48,49],{},"增量同步","。",[10,52,54],{"id":53},"核心概念異動欄位-checkpoint","核心概念：異動欄位 + Checkpoint",[14,56,57],{},[58,59],"img",{"alt":60,"src":61},"增量同步示意圖：Houses 資料表依 ChangeMark 排序，上次同步到 1002，本次要同步的範圍是 ChangeMark 大於 1002 的三筆資料（改價、新增、下架）；每一輪同步排程先讀出 Checkpoint，查詢 ChangeMark 大於 1002 的一批資料，逐筆發出異動事件或寫入 Elasticsearch，全部成功後才把 Checkpoint 更新為 1005，失敗就不更新、下一輪重做這一批","\u002Fimages\u002Farticles\u002Fincremental-sync-watermark.svg",[14,63,64],{},"增量同步需要兩個東西：",[29,66,67,76],{},[32,68,69,72,73],{},[18,70,71],{},"異動欄位（ChangeMark）","：資料表上的一個欄位，",[18,74,75],{},"每次資料新增或修改時，它都會變成一個比之前更大的值",[32,77,78,81],{},[18,79,80],{},"Checkpoint（水位）","：記錄「上一次同步到哪個值」，存在一張獨立的資料表裡",[14,83,84],{},"每一輪同步的流程：",[86,87,88,95,106,109],"ol",{},[32,89,90,91],{},"讀出 Checkpoint，例如 ",[92,93,94],"code",{},"1002",[32,96,97,98,101,102,105],{},"查詢 ",[92,99,100],{},"ChangeMark > 1002"," 的資料，",[18,103,104],{},"依 ChangeMark 排序","，一次取一批（例如 500 筆）",[32,107,108],{},"把這批資料同步到 Elasticsearch",[32,110,111],{},"全部成功後，把 Checkpoint 更新為這批資料中最大的 ChangeMark",[113,114,119],"pre",{"className":115,"code":116,"language":117,"meta":118,"style":118},"language-sql shiki shiki-themes github-dark","SELECT TOP (@BatchSize) *\nFROM   Houses\nWHERE  ChangeMark > @Checkpoint\nORDER BY ChangeMark;\n","sql","",[92,120,121,140,149,164],{"__ignoreMap":118},[122,123,126,130,133,137],"span",{"class":124,"line":125},"line",1,[122,127,129],{"class":128},"snl16","SELECT",[122,131,132],{"class":128}," TOP",[122,134,136],{"class":135},"s95oV"," (@BatchSize) ",[122,138,139],{"class":128},"*\n",[122,141,143,146],{"class":124,"line":142},2,[122,144,145],{"class":128},"FROM",[122,147,148],{"class":135},"   Houses\n",[122,150,152,155,158,161],{"class":124,"line":151},3,[122,153,154],{"class":128},"WHERE",[122,156,157],{"class":135},"  ChangeMark ",[122,159,160],{"class":128},">",[122,162,163],{"class":135}," @Checkpoint\n",[122,165,167,170],{"class":124,"line":166},4,[122,168,169],{"class":128},"ORDER BY",[122,171,172],{"class":135}," ChangeMark;\n",[14,174,175,178],{},[92,176,177],{},"ChangeMark"," 上要建立索引，這條查詢才會是索引範圍掃描，而不是全表掃描。",[10,180,182],{"id":181},"異動欄位要怎麼產生","異動欄位要怎麼產生？",[184,185,187],"h3",{"id":186},"用-datetime-的問題","用 DATETIME 的問題",[14,189,190,191,194],{},"最常見的異動欄位是 ",[92,192,193],{},"UpdatedAt","，但時間戳記有幾個問題：",[29,196,197,214,223],{},[32,198,199,202,203,206,207,209,210,213],{},[18,200,201],{},"精度不夠","：",[92,204,205],{},"DATETIME"," 的精度約 3 毫秒，同一批交易更新的多筆資料可能有完全相同的時間。如果剛好在這些資料的中間切批，用 ",[92,208,160],{}," 會漏掉同時間的剩餘資料，用 ",[92,211,212],{},">="," 又會重複處理",[32,215,216,219,220,222],{},[18,217,218],{},"依賴每一段程式都記得更新","：只要有一支預存程序、一個手動修正的 SQL 忘了更新 ",[92,221,193],{},"，那筆資料就永遠不會被同步",[32,224,225,228],{},[18,226,227],{},"時鐘可能倒退","：不同應用程式伺服器的時間不完全一致，寫入的時間不一定是遞增的",[184,230,232],{"id":231},"用-rowversion","用 ROWVERSION",[14,234,235,236,239,240,243],{},"SQL Server 的 ",[92,237,238],{},"ROWVERSION","（舊名 ",[92,241,242],{},"TIMESTAMP","，但和時間無關）是更好的選擇：",[113,245,247],{"className":115,"code":246,"language":117,"meta":118,"style":118},"ALTER TABLE Houses ADD ChangeMark ROWVERSION;\n",[92,248,249],{"__ignoreMap":118},[122,250,251,254,257,260,263,266,268],{"class":124,"line":125},[122,252,253],{"class":128},"ALTER",[122,255,256],{"class":128}," TABLE",[122,258,259],{"class":135}," Houses ",[122,261,262],{"class":128},"ADD",[122,264,265],{"class":135}," ChangeMark ",[122,267,238],{"class":128},[122,269,270],{"class":135},";\n",[29,272,273,287,294],{},[32,274,275,276],{},"資料庫層級自動維護：",[18,277,278,279,282,283,286],{},"任何 ",[92,280,281],{},"INSERT","、",[92,284,285],{},"UPDATE","，不論來自哪個程式、哪支 SQL，都會自動更新",[32,288,289,290,293],{},"在整個資料庫內",[18,291,292],{},"唯一且遞增","，不會有兩筆資料相同的問題",[32,295,296,297,300,301,304],{},"它是 8 位元組的二進位值，程式中可以轉成 ",[92,298,299],{},"long","（",[92,302,303],{},"BIGINT","）來比較與儲存",[14,306,307,308,50],{},"如果因為各種原因不能修改資料表結構，也可以和 DBA 約定，在資料異動的流程中一併更新一個遞增的異動欄位——重點是",[18,309,310],{},"每一個寫入的路徑都要涵蓋到",[10,312,314],{"id":313},"容易漏資料的地方還沒提交的交易","容易漏資料的地方：還沒提交的交易",[14,316,317,319,320,323,324,327],{},[92,318,238],{}," 是在資料",[18,321,322],{},"被修改的當下","就配發的，不是在交易",[18,325,326],{},"提交","時。想像這個情況：",[86,329,330,340,350,360],{},[32,331,332,333,336,337],{},"交易 A 修改了物件 H-0877，拿到 ChangeMark ",[92,334,335],{},"1003","，但",[18,338,339],{},"還沒提交",[32,341,342,343,346,347],{},"交易 B 修改了物件 H-1003，拿到 ",[92,344,345],{},"1004","，並且",[18,348,349],{},"立刻提交",[32,351,352,353,300,355,357,358],{},"同步排程執行，讀到已提交的 ",[92,354,345],{},[92,356,335],{}," 還沒提交，讀不到），把 Checkpoint 推進到 ",[92,359,345],{},[32,361,362,363,365,366,368,369],{},"交易 A 提交，",[92,364,335],{}," 出現了——但 Checkpoint 已經在 ",[92,367,345],{},"，",[18,370,371,373],{},[92,372,335],{}," 永遠不會被同步",[14,375,376,377,380,381,384],{},"SQL Server 提供 ",[92,378,379],{},"MIN_ACTIVE_ROWVERSION()","，回傳目前",[18,382,383],{},"還在進行中的交易","所持有的最小 ROWVERSION。同步時只處理小於這個值的資料，就不會越過還沒提交的交易：",[113,386,388],{"className":115,"code":387,"language":117,"meta":118,"style":118},"SELECT TOP (@BatchSize) *\nFROM   Houses\nWHERE  ChangeMark > @Checkpoint\n  AND  ChangeMark \u003C MIN_ACTIVE_ROWVERSION()\nORDER BY ChangeMark;\n",[92,389,390,400,406,416,433],{"__ignoreMap":118},[122,391,392,394,396,398],{"class":124,"line":125},[122,393,129],{"class":128},[122,395,132],{"class":128},[122,397,136],{"class":135},[122,399,139],{"class":128},[122,401,402,404],{"class":124,"line":142},[122,403,145],{"class":128},[122,405,148],{"class":135},[122,407,408,410,412,414],{"class":124,"line":151},[122,409,154],{"class":128},[122,411,157],{"class":135},[122,413,160],{"class":128},[122,415,163],{"class":135},[122,417,418,421,423,426,430],{"class":124,"line":166},[122,419,420],{"class":128},"  AND",[122,422,157],{"class":135},[122,424,425],{"class":128},"\u003C",[122,427,429],{"class":428},"sDLfK"," MIN_ACTIVE_ROWVERSION",[122,431,432],{"class":135},"()\n",[122,434,436,438],{"class":124,"line":435},5,[122,437,169],{"class":128},[122,439,172],{"class":135},[10,441,443],{"id":442},"刪除要怎麼同步","刪除要怎麼同步？",[14,445,446,447,450],{},"增量查詢只找得到「還存在的資料」。如果資料被 ",[92,448,449],{},"DELETE"," 掉了，查詢根本查不到它，Elasticsearch 裡的文件就會一直留著。",[14,452,453],{},"常見的做法：",[29,455,456,472],{},[32,457,458,461,462,465,466,468,469,471],{},[18,459,460],{},"軟刪除","：不真的刪除資料，而是設定 ",[92,463,464],{},"IsDeleted = 1","。這也是一次 ",[92,467,285],{},"，ChangeMark 會更新，同步時看到 ",[92,470,464],{}," 就從 Elasticsearch 刪除這份文件",[32,473,474,477],{},[18,475,476],{},"刪除紀錄表","：如果一定要實體刪除，用 Trigger 把被刪除的 ID 寫進一張刪除紀錄表（同樣帶 ROWVERSION），同步時一起處理",[14,479,480],{},"房仲物件本來就有「下架」狀態，用軟刪除是最自然的做法。",[10,482,484],{"id":483},"失敗時不掉資料先寫入再推進-checkpoint","失敗時不掉資料：先寫入、再推進 Checkpoint",[14,486,487,488,50],{},"順序很重要：",[18,489,490],{},"資料同步成功之後，才更新 Checkpoint",[29,492,493,499],{},[32,494,495,496],{},"同步到一半失敗：Checkpoint 沒有推進，下一輪會從同一個位置重做整批，",[18,497,498],{},"不會漏",[32,500,501,502,505],{},"同步成功、但更新 Checkpoint 前程式當掉：下一輪會",[18,503,504],{},"重複處理","這一批",[14,507,508,509,512],{},"重複處理是可以接受的，前提是寫入 Elasticsearch 的動作是",[18,510,511],{},"冪等","的：以物件 ID 作為文件 ID，用 Index（存在就覆蓋）而不是 Create；刪除不存在的文件也不視為錯誤。這樣同一批處理兩次，結果和一次相同。",[514,515,516],"blockquote",{},[14,517,518],{},"反過來「先推進 Checkpoint、再同步資料」是錯的：同步失敗時，Checkpoint 已經往前走，這一批資料就永遠被跳過了。",[10,520,521],{"id":521},"中間多一層訊息佇列",[14,523,524],{},"同步排程可以直接寫 Elasticsearch，也可以只負責「找出變動的資料、發出事件」，把寫入交給訊息佇列後面的消費者：",[29,526,527,533,539],{},[32,528,529,532],{},[18,530,531],{},"偵測與寫入解耦","：排程跑得快，不會因為 Elasticsearch 暫時變慢而拖住整個同步",[32,534,535,538],{},[18,536,537],{},"多個下游","：同一筆物件異動，可能有好幾個索引要更新、還有通知服務要發推播，各自訂閱事件即可",[32,540,541,544],{},[18,542,543],{},"失敗重試","：單筆寫入失敗時，在佇列中重試或進入死信佇列，不會讓整批同步卡住",[14,546,547],{},"這時 Checkpoint 推進的條件就變成「事件成功發布到佇列」（並且開啟 Publisher Confirm），後續的寫入可靠性交給佇列的 Ack 與重試機制。",[10,549,551],{"id":550},"其他選擇cdc-與-change-tracking","其他選擇：CDC 與 Change Tracking",[14,553,554],{},"SQL Server 本身也有內建的變更追蹤功能：",[29,556,557,563],{},[32,558,559,562],{},[18,560,561],{},"Change Tracking","：記錄「哪些資料列變了」，不記錄變更前後的內容，負擔較輕",[32,564,565,568],{},[18,566,567],{},"Change Data Capture（CDC）","：從交易記錄讀出每一筆變更的前後內容，可以搭配 Debezium 之類的工具直接串到訊息佇列",[14,570,571],{},"這兩者都由資料庫維護，不會漏掉任何寫入路徑，但需要 DBA 開啟、會增加資料庫的負擔與維運項目。異動欄位的做法則是最容易理解、最容易除錯的起點。",[10,573,575],{"id":574},"使用情境房仲平台的資料同步","使用情境：房仲平台的資料同步",[14,577,578],{},"在房仲平台的重構中，買屋、行情、社區三大中台的讀取全部改由 Elasticsearch 提供，資料同步分成兩條路線：",[29,580,581,587],{},[32,582,583,586],{},[18,584,585],{},"全量","：由批次程式把 MSSQL 的資料整批轉入 Elasticsearch，用在第一次建立索引或 Mapping 調整後的重建",[32,588,589,592],{},[18,590,591],{},"增量","：和 DBA 約定資料異動時一併更新異動欄位，同步排程定期比對 Checkpoint，把有變動的資料包成事件送進 RabbitMQ，再由消費者更新索引",[14,594,595],{},"經紀人改價後，前台通常在數秒內就能看到新價格，不再需要等待整批重建。",[10,597,598],{"id":598},"一句話總結",[14,600,601,602,604,605,607],{},"增量同步靠「每次異動都會遞增的異動欄位」加上「記住上次同步到哪裡的 Checkpoint」：異動欄位優先用資料庫自動維護的 ",[92,603,238],{},"，查詢時用 ",[92,606,379],{}," 避開還沒提交的交易，刪除用軟刪除表達；永遠先同步成功、再推進 Checkpoint，並且讓寫入是冪等的，重做也不會出錯。",[609,610,611],"style",{},"html pre.shiki code .snl16, html code.shiki .snl16{--shiki-default:#F97583}html pre.shiki code .s95oV, html code.shiki .s95oV{--shiki-default:#E1E4E8}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html pre.shiki code .sDLfK, html code.shiki .sDLfK{--shiki-default:#79B8FF}",{"title":118,"searchDepth":142,"depth":142,"links":613},[614,615,616,620,621,622,623,624,625,626],{"id":12,"depth":142,"text":12},{"id":53,"depth":142,"text":54},{"id":181,"depth":142,"text":182,"children":617},[618,619],{"id":186,"depth":151,"text":187},{"id":231,"depth":151,"text":232},{"id":313,"depth":142,"text":314},{"id":442,"depth":142,"text":443},{"id":483,"depth":142,"text":484},{"id":521,"depth":142,"text":521},{"id":550,"depth":142,"text":551},{"id":574,"depth":142,"text":575},{"id":598,"depth":142,"text":598},"2023-09-12",null,"md",{},true,"\u002Farticles\u002Fincremental-sync-change-column",{"title":5,"description":118},"articles\u002Fincremental-sync-change-column",[636,637,638,639,640],"Elasticsearch","MSSQL","資料同步","ETL","系統設計","0ucwWycVFEurSGHnDBTEuXf_VWqkLei0CEm53b8Nyw4",1792959150518]