From cce66f6107469dc9c2bf4d957f600f0fa79c7389 Mon Sep 17 00:00:00 2001 From: fengmk2 Date: Tue, 20 May 2025 03:09:22 +0000 Subject: [PATCH] fix(server): add stemmer filter (#12358) CLOUD-214 ## Summary by CodeRabbit - **New Features** - Improved search functionality to support stemming, allowing searches for variations of words (e.g., "window", "windows", "design") to return relevant results. - **Tests** - Added new tests to verify that search results correctly highlight and match stemmed word variations in document titles. --- .../__snapshots__/elasticsearch.spec.ts.md | 71 +++++++++++++ .../__snapshots__/elasticsearch.spec.ts.snap | Bin 4181 -> 4379 bytes .../__tests__/providers/elasticsearch.spec.ts | 93 ++++++++++++++++++ .../src/plugins/indexer/tables/block.ts | 10 +- .../server/src/plugins/indexer/tables/doc.ts | 8 +- 5 files changed, 180 insertions(+), 2 deletions(-) diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.md b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.md index 115844909..d1c77e73b 100644 --- a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.md +++ b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.md @@ -423,6 +423,77 @@ Generated by [AVA](https://avajs.dev). }, ] +## should search doc title support stemmer filter + +> Snapshot 1 + + { + _id: 'workspace-test-doc-title-stemmer-filter/doc-0', + _source: { + doc_id: 'doc-0', + workspace_id: 'workspace-test-doc-title-stemmer-filter', + }, + fields: { + doc_id: [ + 'doc-0', + ], + title: [ + 'Deploy on Windows by a designer', + ], + }, + highlights: { + title: [ + 'Deploy on Windows by a designer', + ], + }, + } + +> Snapshot 2 + + { + _id: 'workspace-test-doc-title-stemmer-filter/doc-0', + _source: { + doc_id: 'doc-0', + workspace_id: 'workspace-test-doc-title-stemmer-filter', + }, + fields: { + doc_id: [ + 'doc-0', + ], + title: [ + 'Deploy on Windows by a designer', + ], + }, + highlights: { + title: [ + 'Deploy on Windows by a designer', + ], + }, + } + +> Snapshot 3 + + { + _id: 'workspace-test-doc-title-stemmer-filter/doc-0', + _source: { + doc_id: 'doc-0', + workspace_id: 'workspace-test-doc-title-stemmer-filter', + }, + fields: { + doc_id: [ + 'doc-0', + ], + title: [ + 'Deploy on Windows by a designer', + ], + }, + highlights: { + title: [ + 'Deploy on Windows by a designer', + ], + }, + } + ## should aggregate query work > Snapshot 1 diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.snap b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.snap index 48600af4da0d4987592e1d577bada58ba3e06322..97ef5b13df0c81c251f18bcf30c617b4600a5bc8 100644 GIT binary patch literal 4379 zcmV+$5#;VcRzV}^RtP>0(i?%(mHqafq z`$IoT?;nc@00000000B+Tzixo)p`HjJ2TSgvCR+NK* zBybLK%E?xbS+7{FBuiRshfp9Sp>S#ni36mhJbVHH0=5t2fqhI{a1-j7ghCyXlZ4U| z9J{1(pf4VIP~VYeG`cgR*=4cuIhFtLj_&>4JKz20{_g$0?|w7)PNZ^$^i=sTwkwrv zC8um&U7jq= z{>;6ZOtO+(?|)H%8|)vEi8KUY4S>x6ZUyilfK!QU0dZZNxSLzwCg48p53f?#vX@=Vy94RTDT&Ap+{}HK|dndCSCUe;hla(@)UPuvOk^ox?CpB%9S|+}~CBQ!r;9m*w z76BGfAVPtQDNv-qEflzq0#8z|D}nEdzDR+e^RR$PPn6S>D^o}tROPB_hXeU_1sb4! zqgK_vy{c+{OAV??W-{4IwvbQe)Vk$@LC@(zzM|wSYDL0O;Z2v6WJSr0C#!W+hPs-# zYWXdO^7iRuX)06Llpimalq<8!CKd5b10q;VmXv&DTz&3Z7e6ro0yV%yE_r2Prli)t zW2hHO%7j|DQZ&d~QYOX?GF@n_n<-}WakPOD6;TqcR3H=jH4%73tUW*}UMeNG5K+4d z27@YSzX&|bfg~REfD72+Y6Q7>(2rceiyX+!gZ|qEyv>1R9<f$(S| z9PLss`og25>>`>7$9lqJ{o!bTINl#&_lbnZ2E(yW3g~_ zFdSVQ9!sz)b`cGaj)h~rdM6AM>t%(k|1q_sk986qWtdnOD`X|naP(aD7V&VbkM$Ez zsNlhHY?Mh@Z{E+Um>A}sqij&IQMGxuDo=kn8VyGWY8|p$46vKV*0P>Q`@-=a>j`jr zPghTWq^md56}_};V5qBasJnM{&tUWu)^L59wW`$DvX4}kGV2+otyiP^KGp=3sIF;M z5amIkt2$J9I)7WTvLem5$Vc=*N>L!RN( z2%Fc#ov9IC3zU%I)QCzfRz1|&Sq0bl#mZPom1gW$WH>caWt&!~l1}>(PiP=UL9J9} zqE?|1QvDqsPK}HWuuFC|n$WJIUD}nQwoktrHP=Sy;g4<*W1SQvN0VO)ytxKFJlRF&xYElUOT4u-0M8z#{1S= z)O#`La?GM_pLvwgQS*IcU0g4kdIe3rmy*Wr^Z`ZBwNP$deSKyX#1j^R)*qWLUVZ8~ z46@(cv*Dr6M}76jmN3g_7*V&yBdMewwjtnZE?A{VH5`4>MiS!Dz?_Fy}Hig zDxqp#j_0(Nm(%ic{$G1JcZ$HhPF~IvBCyZN%lV26*zV-z9B={8Ie9rpT)TKMEzhRq*_=$C zjY;U)dp7f-jnQ>6X(4av%p5mbX<_66r!aDDC#i*#(^pK{gp-BqtPHJd0Nep!*Zd5v zvn&j)NL4qD_SQ3Y_SPFV_ST9f_SS_2xP$=LINDp{E&_au0M8QOFb_A|RG@&1x2fE7 zwW-<~6!Gp%D?!!5GHPQ!olC7vq^z-lZrJFy$jz4h$%H;ef!)->7Wp3v{DwN%B4-G| zGQq(Xi3-4o(9F!RT0A}+Pb9JzjfH-3$Ie$CzxI{KA3AjRo>%tV{K}qN4}X2<;lKXu z;fJ>!*|F=;o^6M>Jz(7f|75H%@`Q06HC6XG2r--*G0I_Zd0I?tF|ozO6UW5E4{tlR zvme>9i*c|ji?+M=H14jMLb_pZRYA@>>tZWD|2cr$0qmR~pF8w<7^x`bN+eTAM^v#Q z=_{rfzaw4hJUU=U=l`{#^YSKi{u2URM1YbborT8;u$utSIaB!v0gm!8*IcPA)yC{h z;U&#fr`;+*h-Aau4;jiOwV{Ex7h6=ygjOiyxbSo}s+BrUC zoy(rb$aZ+!11*lUIMU*Xi6e|1dPmxB)IW>eXmxU?Qb-r3i#esDa2wKf8rg=d$aVvO zdjLE>KeBz;cE%VbGqCQU9qZn)VcmzCud3!HfiF98aVG`7>BPm?De%V0#zpnaNNC18(082$+MC6+KOie!_0)Hj~pB91JL|{iP-|g3Y(^E}NeG;%h0wNM{o&=0b zz_k)^kHj@yY-+k!0-ln9f02MUC7|5}bi06$y11rinI4`?RpPYE{xt9lhB~?7T<+U0 z>xJ%ThH}dTJ>qVDptr(M-*9%P3w(jwUf`nJPT()Otp(oWb`ba_2Z2S|N#F}*dx1-` zoxr!t)&lR99R&V2xw*hDkCVWi9(#c=_1Fo##bYh-T^IBT3Wtq+xeqwo=dh8l@c{`AWC|Bu?gLW3X1l#i z=yg8e1`cF8ME;r&c!UF)4w3iyfc+fEbnN@@KH%3Jh(GqFK0mO4OJ)u5ptJozq-n;h zOz1j4@G%b5&VhU-Kd{*k-0TPL^8H{ZBk&Bq zw{rr(7zbLygOmU;#ex2i2Yo&O{3Qok%7eBalj#8tw2TKm9RR-1fkONUe-!}!D*$+c zjlku6Z=oQtf&+bk2PJ~QIt~=(K~q6sBL`Z+gKh`{TRG6#Jm}#d@F)jb$$_NjgTRl2 zz|kNewF67q8ykGk(asNE)iF^hbPR<$Qpr-6-I1|eJE5>SG;ZcBG#-%KbP%jYnTME_ zo=TQ&!p-V!?0PVcquXvAzS=k<$y{!}L$S#lupEll@-{uDvuw4vmrqo!7WeT9met~Z zQ$nCxJfIaz4aI|6v8$mtsujB%iep-_Y}_^@-CbI_=e>J0UTum+Y!rKZo{TA@K^uwH zldvcoy zKbI+jTcno0T<};f>JGBAQSp!0C)2-KXW+X+W?;6VO|^0{D)SdJH58u%5Rs6 z`vL;60NykKZXm$j1o-<}60C#&{96K03M{2SlmZ{2z(xw(%17;dHty&nO_Ecer*@|N z8#ZybB~JOJafkfU3Bk$t*eTcxd{D3x_$`~1*`=a`z-z_k0#DlIzRHAd6@f20C9pmw z0=u0OSbr!2KjlCs-{W-=c!L9(e2>#4V2RWS;(ZU_ump@sz{e$^AOWA3fWMJ|Z%Dua z33yoo-eNHnE)&+|pR9BNt2vO#Ke@;ST*`q={s{}4lsS;eKe^Eb{1pc>`6ujk$8T^T z-anxSUBFK`(1N*Hb1f_Aq_u*2oeb@H@18at%+AoRjnC$ddG+{9O~5G66pMvYB~-2` z)6+^RG?C3!lu|wJe!X_?b%xcs*Np(S1K2aabFYZ)nN3ZsNL4Ju$=3^ZCtt*BgA5R| zs>#V0ORrjQ6^l2YeC?lode!dVMZ?+GVw*j9O|!Ev#ro*^rrN7DO?RsM zDe$nTxs=IaLr(4FEnX+^0Df%Cb1!4w+#U= zZ$iN932-9;b~v(1c!>aq2=ERM6-@*@y%rdmUygkBeXZELOOCJ7Y|u84a(VMC`0MPl z;2)yEE+-!Cr@*ruh>scwzoY=mQTqOzlicO{e6W5Uv_5q#L!tD0P>B;$Uod0uoT9dDfcak^;vxDP5S!SJYo(jTWCYEG6P( zmMtQUtJ;*Zh2IvOdag3{`Z@HhhauQ;ApssGjVcv*s_iAfQv`U11NyBO`X3Qh_xiK& zQtes%G1Z=h#6F5}B?X?Lz#$6!j$iLg%c7*c_p(I*z90Y(@o8FZrY7GNfM*0?g$SG@ z0+;bQRtx8}(+ufzxoW8x43|PNqR!^XbaHbT@8+OE)6Gk6U^5Rh zWqk^_x`Er>z+N7@$OLU;-&v4>h|EDxGu=EP1M7I$8OP6|ST&akWiWRX+a$Z2k3qpScP}n6bG@QPgjI?WRt*v-?EI>~43~CE5H2^|T~a zu4F4S?7TQMQ7BC(E4l=lPrh)zKDjsrU?YH=0escjAD@M>Yy1?~Y>i!+P!>rB=00000000B+Tzhm|#hL%j+=s3nmK;b}AlaapDx_@`TYgGjuR#hHempu;DMdGzD-*@)RBpODUanO? zdUv6atK~L^o)h2$`$rTa^#eE$z!m^E1Go>ssYG5vz(V|)^RJHa$BeU&+@#5HcsW>)Xt1|9|5dw@8U_0Tsrp;U{B=~m(_!a^Fl>o01 zU>ODC6u6WE(-gRg0{2khG0Kq=cvAE^3jBhHg)MTToSfuBId2k`T$c_93K;@4K|^M# zYP18IXnsdcqRABs#agjEl`Cm|s|1sr^W~|UI#ts;vZjtezM|%8YGF86@0&37HQ3eK zZ!)#FPUb2Th4SX9;pvKcby3}{A---xM5c2Ub*eV3t-DU*rzSwS0T?ait}f41wEj0t z{X#_@)f(4`CSEJ*=&*^Wi_LvA(*Q9kK zL<2o30#9=wi3i;)1H0sAkj#UAECbJRARiC^@K0BIpxDJ1LXVakW$`zq$X{&U;w3JgG)Y#i@ztc&#>F?Qd@{6{pl( zWnFH1x;-~KTAWhj1+`inn`$pst21hQIvY=P^`+XYrCfD9o-a>M=B5hOcx8ER$gm$!sju85`<}C3|A&o;Z6?JT}x9OQmDUZq}YzAIqdzOFA7(^~Hu#v1DH? zxjr_OWnJtc8OsdCQe6fShDmj?Mn-=~Yw2c0k{O0cb+AU(l8hzK*Itp1rMelNbXEiR z#Zno@;q34p*2UPcQcki-r83&^gvL)#ESZcYdm9AVD|*>WQ|lS!Om{5ZX}NO#jJ6>Imo(av29=?#m9nWfY4i>ml)JoV-2C3z5wlcG zvK+D*+ijg?GGl#js)M7Vi&xOadpT()XG|!0zKw8a#n){WK{{*0=x}|wyc#u(~J@}OFp}(v$E>t#_D1g^~2@TrdgO|YmA&-s?>U`^k!w2))cA{ z<@irqjB*yEoImO)=XMde+bha>R0IxqMLA!Tft_Aa&OsS?#w*G>Dg(#7qMV2iSmqVw zbo+pQuPA4u4;c0VH+e-lkNbe{dPOi?SHmEJikqkueiG zD_)mPv%mF}lfU(fgTJ-f#oxM^0GAQqI!}K~+(UqG5a4M79O21?)m$x!r{x9xu6k?UW0t^hwrU`6%evf7pZ^NLtpIi}OrP6~eHgE))mpqz&c`*j;`ys4 zn107Qw0(5YNuB@KL7i8*sPlUWa0vk_p6V<-On|)vc*a|mj}qV*50mGs%2H#_-YUGp zO?3LJ!h}c;n)@zOyQB>?>FuR9oeH5f3Vhg0YguyV7B8)RmI6PaZehBeF6h!ZG0Hk0 zo5z&x$c}p#b!1UT7Inm;BTPFCLb^5TnS(W2mz$}T^X19ul3G(agLH$gZ2fl1whh2t z03KPGvc21J$Cx=YY26_wt$V{k>)z#}b?+m<6$H4(TkF0}fP)11g}2tdP5|&u<_Xlg z*qmf&QonaNsNaQd>NjQTJjJ?sw3&aF0-yKN#oZM6x|c4#Oo3NUHeJ;2th7t3xf0TK z@tw`OXfD~C&HOt!gsojB{gVd(WB_aca0P%70F!7I6yXIWfdmQJQYR85B2#@H52bfG~PP)UEdd*WQo1_C9LYs}SF-6!Mc^+*;3Fb%iwNv$)VuwLA9|`~Xix%{NI+ZyE|7p> z3AkPY?vl8n%Pm9qOTgn2@GlbZssyykKtcxICv!v3wk)2k32{<(z7710sZVLTm;0t{ zf6)E2sonNMkNDg#^j4eto9^xmhA;9tGd%5cV)%1DdxrPe2{<#{8*pOy zF9P-qUmx&b_@00}!`}-yFsu;z+W<2Z9)^5X5I863VaU%50$C1ZNf%ui1V(~x)?Oj> zh9Izw16eMSzY+u<wpm*?~Yy{ZAfnq#pA_8pUK&yGswg|AD1D(Tz9*6)BaiBFENP0E`{3HS# zivUt9u%fkjz&kzt{Kz$Jqvdj2f3$5RSLtxNGFBQl6b_fhZoxwH1-S!(NIlEk&%E@J zT-71nti8sGf_WYZr+EY$^N8n4rG-w#p>Du-DgoQqbXxARwdO9qP_^FN%@dJ9^?aASg9@y>XHZP&MMlA`Tvx*S*0PQMcN`qkaY9mp@6atA^O zfZj&_&NW|9&j(y&1)(ow-H_Sh{rGki#JV)!+OlG&A_2gB<{cZSEEYF`yXw~N4My$V<#7JJfKD$XSfKzX-=&FAG)#qrS zuXL}jy1}Ws>V69B@w(D_k^)atcWWg4>bNJu?!LZ1@8T(Cc0bsE&0u!x*t*#Cd(ado zr4=r-T4r|96|E18YPHDCJ#WW{wR6N-PUrI$`eyxDi8~xC6}2`~nTpoR)6r_aTv4N= z#Y(kiz0C`@{XsnP4;vo2cweLBg_U6b+|x4`h2zTYCjcwJ9N@Lq>hB7irb+qo%v4SL zBsSFIL#Crd951p%r1?~vP`C27z>9KSQ$IEj%7%GkX($3bM4Ck^@T#_-0FM*kDdOg7 z83+B3i6(nvEdo?ui$ACOT1f26c2`s2DGD5>!0-9<&T=eDdOhV<0r-pn+|OVCwOEFH zM*yA@fYl;!t_WPg-`y>pm!;`9_Hw;`z*J69o{np~IX;=&64!Qgy%eKvd>SPgRnfur z+#%ZAo+m{I+vCCL;Jg`4=$s}xn2W={iRr}ba><_Atn_v;+gZQ$lpL77&4JkiPRuGq zcu@jgV*UJ`t7Nh2stioXz+dxF*<#i2kby7pFu#T3qcU(n29EO3umzg(0q6RFjXn+< zvAn$E1GexmOErOTvk$n{2khse%Pi0q_WeBth$|fQG|S7g3b28PopIvIeAdoqLz&zi z)$#K7Q;Ms*vsBd-h`&^%M+3ky4s@oc!|gL{;GFzz3u4W{$Y961g!c0Z(KvH9H>;}l zn;+}gFO+CCoqvD(;eB6r%gi?no9%Z&4Gt=~u}W@w+yOSUCC5tmqWVj{Ugz}H)49Fw z?8}N%Vfd{&gdx5GO!aBNpFDfPEf8_6hd$e&m#6Aa)gQVM5kYXK&;(3T!;? zAz#aT$X0LU2NXC&fny#uwF;p=0chbJX#dG)lUF;Y+%FPb&-!WGU%p@!lLBzN;BnSJ fEC735GYc$hE&x9h+|K&9^)&o{+~8Li8&d!PZ)p4d diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/elasticsearch.spec.ts b/packages/backend/server/src/plugins/indexer/__tests__/providers/elasticsearch.spec.ts index e4968e6be..0d1584f02 100644 --- a/packages/backend/server/src/plugins/indexer/__tests__/providers/elasticsearch.spec.ts +++ b/packages/backend/server/src/plugins/indexer/__tests__/providers/elasticsearch.spec.ts @@ -1210,6 +1210,99 @@ test('should search query match ref_doc_id work', async t => { t.snapshot(result.nodes.map(node => pick(node, ['fields']))); }); +test('should search doc title support stemmer filter', async t => { + const docId = 'doc-0'; + const workspaceId = 'workspace-test-doc-title-stemmer-filter'; + await searchProvider.write( + SearchTable.doc, + [ + { + workspace_id: workspaceId, + doc_id: docId, + title: 'Deploy on Windows by a designer', + }, + ], + { + refresh: true, + } + ); + + let result = await searchProvider.search(SearchTable.doc, { + _source: ['workspace_id', 'doc_id'], + query: { + bool: { + must: [ + { match: { workspace_id: workspaceId } }, + { match: { title: 'window' } }, + ], + }, + }, + fields: ['doc_id', 'title'], + highlight: { + fields: { + title: { + pre_tags: [''], + post_tags: [''], + }, + }, + }, + sort: ['_score'], + }); + + t.is(result.total, 1); + t.snapshot(omit(result.nodes[0], ['_score'])); + + result = await searchProvider.search(SearchTable.doc, { + _source: ['workspace_id', 'doc_id'], + query: { + bool: { + must: [ + { match: { workspace_id: workspaceId } }, + { match: { title: 'windows' } }, + ], + }, + }, + fields: ['doc_id', 'title'], + highlight: { + fields: { + title: { + pre_tags: [''], + post_tags: [''], + }, + }, + }, + sort: ['_score'], + }); + + t.is(result.total, 1); + t.snapshot(omit(result.nodes[0], ['_score'])); + + result = await searchProvider.search(SearchTable.doc, { + _source: ['workspace_id', 'doc_id'], + query: { + bool: { + must: [ + { match: { workspace_id: workspaceId } }, + { match: { title: 'design' } }, + ], + }, + }, + fields: ['doc_id', 'title'], + highlight: { + fields: { + title: { + pre_tags: [''], + post_tags: [''], + }, + }, + }, + sort: ['_score'], + }); + + t.is(result.total, 1); + t.snapshot(omit(result.nodes[0], ['_score'])); +}); + // #endregion // #region aggregate diff --git a/packages/backend/server/src/plugins/indexer/tables/block.ts b/packages/backend/server/src/plugins/indexer/tables/block.ts index 926147755..828d8094a 100644 --- a/packages/backend/server/src/plugins/indexer/tables/block.ts +++ b/packages/backend/server/src/plugins/indexer/tables/block.ts @@ -31,7 +31,15 @@ export const blockMapping = { analyzer: { standard_with_cjk: { tokenizer: 'standard', - filter: ['lowercase', 'cjk_bigram_and_unigrams'], + filter: [ + 'lowercase', + 'cjk_bigram_and_unigrams', + // support `windows designer` => `windows`, `window`, `designer`, `design` + // @see https://www.elastic.co/docs/reference/text-analysis/analysis-remove-duplicates-tokenfilter + 'keyword_repeat', + 'stemmer', + 'remove_duplicates', + ], }, autocomplete: { tokenizer: 'autocomplete_tokenizer', diff --git a/packages/backend/server/src/plugins/indexer/tables/doc.ts b/packages/backend/server/src/plugins/indexer/tables/doc.ts index 381575be3..1647fad7e 100644 --- a/packages/backend/server/src/plugins/indexer/tables/doc.ts +++ b/packages/backend/server/src/plugins/indexer/tables/doc.ts @@ -24,7 +24,13 @@ export const docMapping = { analyzer: { standard_with_cjk: { tokenizer: 'standard', - filter: ['lowercase', 'cjk_bigram_and_unigrams'], + filter: [ + 'lowercase', + 'cjk_bigram_and_unigrams', + 'keyword_repeat', + 'stemmer', + 'remove_duplicates', + ], }, autocomplete: { tokenizer: 'autocomplete_tokenizer',