From 4d256c2e073918d13291ec7fe1ac771e0d33e01f Mon Sep 17 00:00:00 2001 From: Junwei Zhao Date: Tue, 4 Nov 2025 22:22:40 +1100 Subject: [PATCH] Add craw4ai --- .env.example | 4 ++ core/settings.py | 5 ++ data/db.sqlite3 | Bin 401408 -> 466944 bytes k8s/manifest.yaml | 4 ++ .../0035_add_webpage_content_to_page.py | 18 +++++ links/models.py | 1 + links/page_views.py | 64 +++++++++++------- links/serializers.py | 4 +- links/tasks.py | 60 ++++++++++++++++ links/templates/links/page_detail.html | 45 ++++++++++++ templates/base.html | 1 + 11 files changed, 181 insertions(+), 25 deletions(-) create mode 100644 links/migrations/0035_add_webpage_content_to_page.py diff --git a/.env.example b/.env.example index 5558747..d8ecb29 100644 --- a/.env.example +++ b/.env.example @@ -5,4 +5,8 @@ R2_ACCESS_KEY_ID= R2_SECRET_ACCESS_KEY= R2_BUCKET_NAME=home-links +# Crawl4AI Configuration +CRAWL4AI_API_URL=https://crawl-api.junv.cc +CRAWL4AI_ENABLED=true + diff --git a/core/settings.py b/core/settings.py index ea6609d..c1867ac 100644 --- a/core/settings.py +++ b/core/settings.py @@ -212,6 +212,11 @@ R2_ACCESS_KEY_ID = os.environ.get('R2_ACCESS_KEY_ID') R2_SECRET_ACCESS_KEY = os.environ.get('R2_SECRET_ACCESS_KEY') R2_BUCKET_NAME = os.environ.get('R2_BUCKET_NAME') R2_CUSTOM_DOMAIN = os.environ.get('R2_CUSTOM_DOMAIN') + +# Crawl4AI Configuration +CRAWL4AI_API_URL = os.environ.get('CRAWL4AI_API_URL', 'https://crawl-api.junv.cc') +CRAWL4AI_ENABLED = os.environ.get('CRAWL4AI_ENABLED', 'True').lower() in ('true', '1', 'yes') + # For debugging LOGGING = { 'version': 1, diff --git a/data/db.sqlite3 b/data/db.sqlite3 index 0b776cf41adc3868d9e0c41881b60733bc36b79f..39093c1f90d69f10ccda59fd90ef51494e84b1a7 100644 GIT binary patch delta 44569 zcmeIbd301)nkT3QY**NTS-fOAUHObsX@ikMI|3&-g%CDXfJwmZaw(lsAO%t+l0sI5 zz&Nw~B!Pqk+E@dGuml#1F-Tw*ZAj{6&MaqorhBHl|Cp|sGummKu&t@TuROUnE;L2pg4B1*?H@^Z88US9U;q7{qpF8=Qqf4=y$#lFRR7XODOFE4px@!NMZGnOoVVfmZMZzk4w z50;d0Lk#WhQ+v`UUVm%J^@OJq5*Dvcel1bG^9O>97rz<(jg|{f<-7lz#N4Kg2N{VA zG7?vW7M3QiN?LH@)W^?P42Mb+$CDE`;rqs?vJ0Pj>7Ca?ulW*R4tc#vFAe|K<%t{r z;h7}~H3^F^EPf{Rd}-2(XBK|6;D5q{n$R1iNn3`Wb}ihOIQ*|YN$p7sp8flTk3;3< zN!7zu&n|o=arke&Nq_Pb5Sp}n_&@GXdiGy0N?!7JKw`qLm;A$$|7S_)7vClI-_=SB zmn`4A4o^$h;A7QKUHDwPcsTU*&ScNx#eco{-z@$Yi~qaD|7`KZ;$dK;VDajO$%%=D zZ}I1P{#-|&1$q3LOP@c_;m_>hfB&oGA1CSn=m~#yJlu2S+lTFDTf_ADm6_g#$kj%p zrNihPFWv5|Dc@b|EBDs@5>vL%SCL+OsC1^eBRnxUJ<>2eavar+fexeffum@tdG_e^ z_yc2L6a@fUE4AyUdq441*H(E;H7=t+p!z2!jP8!w zqSp-2wR-hTLz{X0%dBr7_DoMq>EYAYr^im%pv~S6-#%s}6t5+M%dPn5mX!!Kq$k6cW)!MfY8(o^VTAQ;Hz?h*2=7oFa+1p&o80ZO) z-!sOpn*B$^wP?6 zKuVXl9Hn3rfWv`ka6rfMi3tFqi6ImQ!@t9 zk$dt~N;oH$=|NA?jNa=rmqsxqW@psKr7W5tP z1ijiJe|;VCv2ssIji=h{0<}sj(<9@?*lBdq=(vk{u;|iLTWgcyM$No?J@ea#dJk5S zdGk_asvBqvkGGn)E*ssQz)c`n=k-)q`GOki0ZQcXM0os^d9lxU*kp{1OpkYjk329s z`Uw|b4H;cuppJQKfJmgfH+=81^A)g$c?n-XzTa0>Wp=iyexVck-RraxWWK_!H;t~d zYWa7Z2_L^{p6ZyM=mq4+wZ`1^Y_t8S@$d{L8lBHZHREcN`FXFJ)brOPQ@vbGPL*ADHjjhYNZD8f1077b*f z1dT)H#q8^z9=TCafqh4D7kRG#qTX9~sHmpGSL1a>@m^P7qwV(9R8)E8RY7g7`(r9u z;^m?xcF_`GofQEc~?`#b_j(_&lEXEjqzg`d2ctY|Y^ zYXZKC%HR_X#wTS<^d}xqrZ7>p!m>3`OyvAmYK1YTWiDKisq>%%uq3m$Ei!a9Jl+Bt zlx+ii-@F8B?J2fc_w(L5sTLW!XP)c)=#7x))#T-~sbX)Hf3LgRXm#=a~pK5A74UA1s9#?aX6<{=9Q9b5$9=Cm9{R$(H{Y9q@Mr|q7DAq1daIn6X z&@C+}qGL3e_Zva}v#WqRV#5Q&NK3x|zchAO~zgGokBFa)77+wg4Un4vZZh49L9G6wz-V2KUAbuw!N-LqbttC74vu zkD6D<;&s;teN_RsEo`{Tsyu;!R>*?OdTf9i5c)|;op)cA7hK$56VNtjS@c-_0SE8u zFdHsLCI(IY<|VsE!0V|it3<^>eN~VuW#-y7%-#lL;F>U9;?Q7-zy`ghw$K-rbE~k% zf|aP555~oHNw%SyMoY6XI68A?3X{gwKorC*bj2;=2L2!ZR`daHiRx~Z zuNvD2I*SHVwD9;tqq|l5Idi@n5*xP$fqYhZ9-MBEFg$SrjC@v2^fQLr#CLR#ZkBuZ zd4NHyL8j)C3M^|+Rn;K~@Y3Hb#*mIa8_m~^I}Nj1w!l<)gFtXK5puw;jh#XDT>isS z9Xb*j9E)@g)9w}>IZaRO#Fk(klvGt!182Dz*7$8SOh4=okDuSRZ98aJrY<_&B&^B` zFd^q1=_AhfdAE7}lJWU*wQNM5jm22d?+3lwtFgpW9>!9ecKJ$45!w zdpovleFwez4NuwY)&6pCRY~wrt@riy+UwQ8s1FKGMZKr$^)=jGqT?Y5x_G3pwu4HGa!j1_5nYWD^14qK$Ekx|MyVMXE`PfdWNm(2Kkeg{kNYSnJG1g_Y zhSd7_D|-XdAQOV0PmgpM-R+nW%!{DgMRIL>#Zr+8o7NKVv^ba(0JIT zin1_ooM~akb3@5Iz1O3>V_F}%WnB8oXgO~TjgmOmbL7v9KKz@8#!Usgi%Q<#wGCgl zzqh$)Tgi@s?L{Jp{7pj>Nx|a{)1$5CxuMyOfBUdcaWhdSNQ1f&9&8{f1(NOLapM+r zjvfFhE!9h1`|AAF8bR^+G+#9f74PwYTv%H^V&5AydGLm()>pFMduYQ-mP*yHm1{J{ zwqrU-87ir-tJ<)VM5+K}DwItRfPRPss{9Aj%WB*QGFPs0=`PSgEqJ6Ctw?j}WtE;F zN&B>)5&KLpgOD2ZzCo=pQEpiI3(%FDKC;qt)7EBfN(03Q>*~vbE7x!-Zy*4zJFubH z>e+8zuk%+iA=g8C0;#GOSbwi8?g`3R355(KBvU;75Sq_x%DhCxpA3Hrdd(pc@D4My@VzzcqkAe1k!q+-t%AFlG@1B{x@riy)s(~<+OsF>l zJ&~bXk?Rf4`rHR7z`L!zynvWTz!%s~+`;{U*SFWd-N72{a+XYh4Wryw76h+>5gf6b;BE;nO-4<4(~OjayfzNBfPF?cx5@;V&ORO9HjZG_BT@=+@9|Of}D38lP#r zZw$7wI&~BqNBFDrC|l>>>kkIfgP#PAj;_eyAV5hq^Wr7*=H#59^eC(m2pYhZF?k%H zP$^j`=rATjO51FFfTb=1kJ)-wlo@mU3f8;@*|^Xz4aN*}>{|GAA1%D`@#%>((gL(E zY~kV4cXWGT&`e3EZJRQWHv?dHy0&_?G$KVpi?9c+&x43`N?*C7hN6g=kNe^HWJJ=HrQm;@Y>r}Ldy#ud$3ZhBQi0{ zQX)t_*9n1*W~C>DFNEx=i;*F+bKnnAPj955nYj4Ss35?Ct`E*M^kebZ^ApBWgB_st z>`aKibo3bmcVt!~_rAa!Mq!eL0ARtkW$qvWOh1G+N(w&U%``O8e!v5t`J!{?{FkT- zGXm@&42XpU{Xy)E+T;L(L2OUSn1iN_=4K!mYdv!9i|~bbnw_h2GBHN;0|omMWZkjw z&8wIUV?sX#!1Dj*y(sd$(rt8M= z*VwPITmgZ0@UDddTJ3W31#!J=3UCQRcY`SD;{a$Rq@72O) z>8~lV82Ro)fuOf~l?9mXCz$Y=tE~#|){)@| z>^0lOpRwpA&l|R$ec)sYZ?p>-jSx`t&6HE_#UjY zhTh!@wxQ56{jiaE;b0FG8slMu(RBi}BU3?Y8S^@^P(`0J)5RhK6#Jpt>H0<^gSYUI z`U>(paYc*VyA*C81UGR(u8-WlJUy;M>O4OE@Lc#>1DJ{Z9yVf<-Gppgl`R1V$;jX( zdRVa6UmtWClY=k@qulh^82*B=LCAoDEuM8}u@qYy#g}Fq8Q9IXQ{n!5ics6H>1OK` zo<^q58J&Hq0=|6Rj)yaskC^w5o4q$66h-deHo8wx$;oCAsO%T8F;9BUMeww?;D zT%{FjwnCG}3Jqiu)McYip{g}U2@A;VR}51uG1(TNYz4@AY}Ho3xe6KoXZ2u%su=sy zsvh%HfQY?>^na`EpIaatuWSI(XI!0?vouJ&HNmt1uV)L&pDQSHl(2x!e#I~a+P*p% zoe%o!_FDkAD}eE@EogDi8B{GZo;1udah_So`ZERcQN=9Kk9f~bK%cg!U+h-_H$Sr~ z(+@7mrYXqfY;3Vcr8|*rzqIPcRDse$OnH|#0Lwf?dj7Qy!aZ~t{$2>lSo81GAm5im zBjIP-`<0r%@-@x1r_x*J)ed?Bnx{td*46pzw1btts#LF51vXMnXO$QFlUC=Y zM_RBFgiYQ{Mcf7PcI>y|eA)mFsl3{o=0A|O&sXJLmCmj14tnaatX8jHm%eU4cwT^h zD=^rrEN1382wbhTI$Zl@i{OwUZu(6&KU3H;|_W%ppX&(?!#_U;4n9W{5t=yVaa2D|1)MV-dTX6R%4(s=qq#ksxf8m z)S|r9{B^18vQvxlQ#a+N=5I+Y%1Pa{IdxqjJ;~2V&CgB$b#28)*$OcCp0fR@7|=4* z&~?&R18&t|t$O`-owwG{4xtUUKVy|^haZHZ>hod=RC>#W#XgPzA4z}D%_6{Al=CP8 z#7q`K$F1}2tq=O^Fq@fbOz+M@Uqzj#wvuhCc5$AEZ5}c>0UwO~Vr{25U)N=)uS>ON zHD?wyS=(<{Z4f>!!bh?8?w;-2Ab@AtHTOTdW@WItO6t8A#3F!+YF#Sk2rBG)7+3@Q zeYF7~rp{kqQK@;WYlDZha$g;Y#9!z05}*04R_r7MYcgjw2>+W`ybaFc*z07~KT%tn z>$lKyp5)^%<}dhGdu^|Z!wz*hkh_t&_J;Q*?FCryoR z{jAYNSA{rG(u#JF4~4KZJ#veDH0-8%BzEHKvjfIFcE`Nc2G<{ypm5VD^g^TMK4c8z z1hiI2C#{I;$bnBC<{3!+@PGGS506bvk2JuGAa29yiBmH-+F1x}H*a2!Om)JP7arGd zu?65dImj{cZ+b2%@yfi~1ogx9O#=iuVK%7yO~XYE<~P8FbuB9Fkd)O*HX_($$)sYQ zA0u1G#3{`wbcx-DI?Yl&*(?Vi=n%hV&JWlU*zY5NLAD~rsulVM*&U6;U6ICP#=w{8 zuGv3K{-2{`BL9ga1YI$8tE(j8BJ`p8I@C6JyrrCGT$ms)!ob<=j5ncpkRxhf(ztOF z4*5Z>3XREDSaYW*T4pYO4u`aHWx_lgf>G3b*fc%HTK1iXgvPF>lT2-=`@X)JQ@`pi*y#L!2CfsB|#r^$SM zXB?pr*U(KoRnaELuRtIEuCJ%TXuieMcFs5kshX;YlI0EOH@Q8H{5s?u==`f>R1EwBP{ za?$86+@nPYm)^j^hcCC`3by$aRR&WW&#m24Ly;mqr_#nFwCBqXJhyZ z5D%DQ_#uGYIkxTTy=Y#jlaPJ11)dhHUw}ZAPvjbmEhlNoD*L0XAjShVVKy`Zt;Vqn zGQ#kUMsO_S!X&u3c)SR!aKEx^7-k!CPQ}fueyc6qcMA>8Ts~n>7uk8fou?|S-P#(u`HeDo7i?_PH4WSC`bXeYl<=(Yy$&+!*Quc@Xk=qmfl63q+8S$Gl=<9LxupMTcg- z($AS~UmADF9&WM`MB$?k(oAP~Y)AxLnzf4|)FGBJh)6+faAZk|Sg=6*BG7;&M5aND zSTx~&unR;795F|l>e=5o;BhsbSrm7}w2(t%hr`{C-d=b@anafD`3M(De)v2UoE_%wZxf?+joFEhzaJgW7FejjjnUtN0Rzxj-CKj z&s=VyjS(0USD`Rli4xqX&741iaiB0Z;mG7+9npS~;ln7s4NfcZ6+{NF6SEn?+UwyM zG;)~2)n5Nedb!^nbhB;3otcrbE-OD{qYI*k(fSp~2~ewo0mw7vMY7jH25_PmPXvwd zHBmowhbLHK|-ri!Xo@6h!jEv2#fTJrQS$giY9ulFyS}avMem3>1+yNg z56|I|KXZ?G1wX(h0s4TGR;RczjWG|(2bUmwycKe5wx2aS5xfT{COvm9U)Z#rGF8D= zBKGbU0OgQ38=}qsWpk0v0wP$lF@U&2okT*rkStaV$^nlfeQbELFUe*iJ5m&wB@a6z zSLTAaizPY%N&HXPr$}ldSrW{aW`sUqEn*r8%W%fxCkvWV#_l5FmS*PWC5a1@F{-#Q z^VXG_8v{_VX3jk@2k#@|IehIleqy&F&(0~GLM`zdbPgg9kHahZfPPNiK=>&(Fz_Yl zeQ9ZFzm5BcrEbOand+^@51o9j?%)hpRj)NabWU$u~%wgcZt>!kUSIRypLBau%;I6)6(b%=hV zO27bDUoA13RDgY7y0!EeDifK&Mjg#X@yx%C0fnD-CwXYntS!7iq#+rCE%9z7&FHNU@LTnr>r#V5T_ASp2ls+#BxZb|wCG?&2NCYwbpinkBtNo-W_QbM zK#=w+Kyg$91i3TAut1&$~%pGs?f>ObJ_aOVDkds$nEzwrZ^CpC>DC#717ZOf|$HmpcPpmW7 zjyDO3P6mRX#xb3wT?zgWUIVW@c)XHNnH~g{q!A%doEcDj3m_)uQzD$J28k`neikE# z^q=4m63dI1^=KEQ+x5-q-fvDf;IFKBAzgRqwZC7u{H>Iz{Q<_6nGl@A6w4^o#`>(UpM8g^Z&U*dXnqrxk|M)6=Qk%hWnt z`{QRxjJp^@ISYr(C!ILVM^TnkWq+YLgm%de_9sB#>yJgC11CRu3{IlB#yD+U+k>=* z@W(}6^&?Ri8$U@gBr$&wr8t-1@&riyK2(TPBQ|E@s8HzKA_aByD5i_}LBY5P1SIdV zCh>cql!u4{HU=-lFo^RUv+yl5r~rlD`1^&6>FCN_#2vfRINpycW3@TDfx3NT;Y$s& zFzDQnwM1j9gI!j8fZ_|gII&DVXIaik1?sCYDvijvT?5 zaL$&O_-heqkMLNE=|!(oG>VNBt65~I15pI5e=5V#9J8Pk(-joNP9Xz}c8yQ!2w%e1 z6bH!02G}ge{W~L#sZ*vSYFU&}0pgDS>Y} zSW<%C$Ca>qMIdJl4!z=KD@}+(Yy;qldKQ=wWl;G2a+<5IZUhxwlQG= z@>D$aJMQxH?mV=T)&5TN?a;&G7IW;BBa(_vy3pYWb@+qv0SGKGiQqLS1Ph;J*e}6I zy>;wkLv=YKkizauk~64Pj_9Wc_99qHUDv2odLT9vE)U?zUWOdV(x zOopx)Qz218z@-If%nFIF4#+trbdu1XMh7sfKej?byZtO7&ORA!O(!(dwGJpWx8hL| zXN;I(W{=3TkPnxU%(w!mkiuvhhslhwu36ni#r zSVyN>{WWR?DVpv)g=P;~C^42EDN>4si)zGpRan~_k+Wqp62u$luiIUSg+Qlwg8p(3 zHfQg_fN&>TQFJEm7#sOkxzanD97MJ`xYg z_Q1_Q3J=z3jyfWUWpECyNANtyxR~N zSi?&UJc3T+^^xJ~{B=2>0&@>y;Q`C)NBrS!!LI#@q4AMrkurOZ?6ywa!(^(%nripT zg7@)!href)uAc7=UUksSaH4u)BiRp$*trL6Q;)!HnMtLNYodoWHtjrZa}0Zwbo z9PNkEVPBgyJ8pR+ESc5raLP(eS1^5NL1vRQ^Sbl%hW1GG;Bn#*sx% zb>5Hb5%tC=W9itH!jko0iVD}0$ZupVfm0HT!DXFmgX^fLK@-2Cc%k69lWY%g*ufcC zO?Gs+Irv~0d9Rd{Kx7=upV@s2X9>vb#U46DVM!?o7P5~#mV2xHmYb`mnD$egXX99F zUX|KrypM~|kxiG%aFH}BB4M$x!y`?HeN2W$MNC9{dLba8GO_Hy$}H zJZ10gMpy+E7awo9trhAK7e_SXo9dm8h{sC>vgKpJd!T zcOVOX&RtT^`MV}3HMO*w_`D^Wgko;UPS4DoFU5&YIZmu|&J=6|=RfSPHVIknvt3C& zRwkt@=IOZxC#`|uDQe{yVty6j%SW_U1!NEh*QN)#IC zY-apiWN<+GR90Ira|oQ;B#3B)$Pk1|5{e~i1>RWa{sEUa9QLG6-9?PnG0KzD(qXn; zqqBeQN2m4if#~pY22h&XS>-u&u#Q}(=KU_XsjwS~i`+s7qaf}bje!sHiAW_!Z)pA@ zKPhN7!pacJTJ5W^cIh_m5e%c0e@DWb95{&NDReGlsAcBl6_+GtqJ=j;;mEBNmxJli zDVV|!&m!PJ)zmrllca6pDmWb7e8ptXv)v@g1x?qdNBd;iE3F#w48wPbDX=7iH7BNa zaTH^Ma1aYKjT|!|RTw*Dj<5=N`m8(;dTbIX8LMmarjmEx+r3BIpslo%!>pvIJKx(y zPqVW#^62OLyNh;FAwC{Wf!ma_M+z3~-u=OQyEfZzYmqa8On3a24tL8N9oc>$Z`$<= z-!0g+yNFw^5AMsa-kY@r?oC-r|48?@dffitosq_Y=-C{ei z7=>jEkz?yKhxG}8MU5ycxERP>TX*c<`p&z1N{V*8vvo&N3BZ-$sJcBK>o9nHIqM)h z6?7g3ul6ItzS56_-h2|=s-|1;qo>}f=6+Z=)AnbjWvoT|HXJEFfJo()=%T|H?1DkT zUOFi_UYZpZkE7zXm6;1!ktj14xwAR}b+oBJtY4O8QGAcFVb6zVNRTlUz@S3X&F{ zeDO~#C7x=}rx0^rTZs}?h_lxm*dxX}mk3SV;W4Mdqw@4_-IB@U z4uhA}*C2sNUBFXCl9AOvgs@?=M|=#093T4}2d}|)5o-g+wbIcwVv6jECXEAh?TnUG z>`*3lQ<1)Fj|wwqmhj{~k19Bukl3P-w1R*)ZJWQ&6E~8${zE{)QCJED?xoNi9RolF z)XeGg&H;e2vYw=E^Y1E{)%z#LZnTepmNjSRbO&}}?Lhc9lxpxm^vM{wfmmv1w-CAa zNNwkPO?y<0(pit16J^olBHGHpfoh^bp;8f3paxJ15Y7>Z%m;`sh;~)w-$y-EP6*jDVIi#BwM0fNSIS>8ozOg z<(vxP&+9ZX$nl}(+=(yR+c|HDl%&EGgFkm+n#{PuoKj>C$ht@baiy&}djxnj*Kw6B z^U>UuXu_ivI421q4&x`xDc7b=Gz5#WxJF}R!+IPHtXre8X^Fo(u#0JsS#eI4H8+GV zh}9gL>RdM}z=GQ6KQ!Xl$>f|$Oe5a9l6_up`Cgn`Q++Kk7SNlrLGdJwf99{zZ#x1F4)3sVRI;Yl2 z=cSS!yHu=9JB5sd*mXjAcPOJ+oDM1^KWizhQ53ylhrh<_n7I@!T~Ehe$-yDE03@+F zJQ;)oSv)a|7%OM4PQ5{=;T+4+`-!)#9*! z_K*5HBwFS59n;11IVqZH8KMv~Jtk%yXLYUF)$CT4!AA^P)@FgQ*$=#X#Y~0y#L*dS zhAO>|m6L%UMWJwgQ6iv^z|RFeos1wNNwZR+lU;(2pK)-SYB%&i^c4|OC|a5OKoS=- zVzIC(TMH^{r7^zJcqd<}CRQ*Uc2V^uJrQ+Jr+{FUZWRguhd({+!4RNqq7y9qHo)FX znMzo1f@x1W1k=pGMU|9*@^N;Ato*ccTux%rCN3j512rW%fgGCyKe&q29B1g6-aZJT z;e$kQ$L*KsFjySuzzu73lCmOl@2b%?jFNn|Mn+M+`sjUy-NH=tHU(F*s0{$oGA#k3qduS)pjUv7eWEu#n+z&t& zULl6@+L1{6VKj@3=u?|6l90E|ask(KVkQT~_#kZk1m9T1J#)kQ}Y z36gnMzb#6?$Wf9um~zH&o-sIQVKgKQ0g_j8c_0e3BROUGTqfIrKnhpSg2x@7M$9oH zYNf-2myFhvE;(~1+7M#LDIS^gP_0BnFK9_O&tAhRcG1YPU~Fqmpj|&m}t#j4TLv3y)t5kKY1DBwrpM9PB`xH${#k z#V2ws>&NH}7g02=k+u%C6vly~qY%xJ?Tx?>-bchSjIk&x9yE!&q@}~hPIKC`cmh3FNgMG-ugncEa|s>3Q&Ce0!N5TF$^4+?Wbb^ z&LllL!t=>CjlgpRjJM!~O^=;ihYl@rW~%AYe(J0vMHPjlo;e?B`WmwaGzh743KeE0 zU|5nwN-_b-$_54F4~X8%8bxHM1BcA1WKmHw5-5Y%Do#O$1lr%XA67%tR^lwXMOsQ@ zos!^0!Xe1NGt+Py8-pazqC-Ed-aZ)+WL|=+3BAS_8gc8{>B;-yV=(ocrDJ0k4~H*M z)-Nz~9GL+~W}i-D-$t2?SPZEcU0#)xQ2*J4(uC*!Sfx$AQ>((|%% zva|E(sA@FJe)LUVW?^lf1?|JU+>+cJCo~*v%}KAVsj%|kmB+zBw*H*-wd=C7@-k+> z$xqK-yEZ>3vtV;>v`?s!m7Yh(WwU12kT+R*1+_WRKIN3;W;y$W6Sw*E^eH2MJ>KW# z<*&_O7uy2fWap&kXJ_POWdD3~w%sSGk-08ACwF!Yz@j(#IX|z>vidZfx$LQbu}E8p ziCVZMjL(GqoVx$-mi+CK@WM|Q{C~+06MveJhpAf8SpB1A32!wfHvZt1f`UZ}KYk`5 zVNvM)+T{Oj+45JGCRA@d`kO`e^WnmelaC}V-}r-N35|*Sn||PU^S=a=(-N2d=H=%T z3VyIE`aJYveRB2kjW5os5JH822z4Jw&VJ#A<;xSAmNZrzek)pUQK+_J;SY!Z?m+TP z;_|emOA`v}o7}TrKXoWMIdS znSb@v_wmg5VO{E3o6rgt z4c}@=e)*~8@4l)kzGSaGsp#uWK9~3k2+Ea|l(1y&b9-F>)t~&sGv7UZ=&66V;GZXr zBqiPb_<1vNd1H3Y^9jFE3z3H#?fXwce{(9idd1T0WeLAIys)W0etg3iUw>lg#k0xV zS1iq%ztF04$=jE{m>FI4g4&jI$*s$R>5kRH&uO)I6PG`e_87#T3>92R-nOi9%c|&{ z3Gy&!HKG4>0{H)LPb6<%w&)Ek?ZU&3Aq>BIGWo9(Uu-UYeMLgk!p5b|GUGpfrtxhc zcYkm4|CqFN%TMM2>kp0eCGS{KlCbD+pZlL(JUCXIc9eF{Fll9 zK5<2E{_=#zw;NX-*&Qt~yyIGOO49PCm-3ck2{cwj-_RQPQE2Q|^8fk5k)^rwO!eO; z>cqv86Iv1$4<>$!|9*&{>d&t?J)e;I+9C^V!lFf%$40!j>HfM3abDB6*c~r3XMJv7 zdR~5JUPks@Z`NWH&RYxH&MPWVxbfoxP|t?OkGVOX1|jWYNh2el+8diZzAg2uw0}CR ziMF_f)F`-l%l5;#bpScC5FW7xU*Xh8N+`BC!}%)~0t8NI3Qi?bZe^|uz1EhTzso6} zu#k|Ow{C4#PF%NH;DAiAHY?ATNgDs{il47YICOaV;pAwy58r%~`H7XW(9Xy-M`9!y z15Rkg=gL{H(pZsS>UT`qr$^ea{XRKko-}D6`7GwjE~q1ji;WDTB#1JHlz?-l7yKi1 zRG*z*a5kP}Ct2jNjCGk=_!}#?X0Fdg($B2iwV65g^cU|&;wLTWCzBf`+_!QSy4|o% zREDZwO@7)JKe2NVK?=yXE|})um!{yXa((Ep*Knr>V(Q?N*A4<262$pA^%P85Ff`y9 z#grgD9I~S!wl*$aWj>l)MLouK1XvZ79G3x`8YkRL*Bs4~Q3Xvuu2qX8(17drriW@@ zO8v zLTFX1HI)nw{vzw^tksKDS?|K@^gCx+o0VkkiAIyo-J?0IhJ94q>vfP25osm7x?T-r zng1FVNF8%M6IaO{~9T$W49IM)tD!Qhmz`fr;D zvaVr$d1GQ^i(|d12(c9XX-1uQsqZs~Yi=W!S z*JdWb%!`Cwa+(EBn8?Hgc?LMX6!*NqFGJ>aV+vuZl>QHq-3T`2WcPH{Un#B~Ias5m z?a~e&JV^I~d9;vi;sVNU69M1{sNK z5Jy&4<*$d+3^&jrmE#NkFla<9Vzko}UELzdBYHf!-IKt`SiO8TXH&OOU zT6$@|G&duw6ny|t) z1%`;!;YdR_VzS_xF(#WacnML~vf(0anbA-7O^+U@E>B*>HH^|N3ky_P@>rJI(?ST< zHuIb*WOIljF8^pl3|eGpNRCaA=j|v*&CAV-UtdB2>S;?y_<9HR1=#?RXi%q!`{U6| zhX_kFp`3Y5E7-MNE8O`$a`{jyH=;%5Wb0`%86xs4#uJeE9vlGBVdOW1D-q2j$tSui zSE3Rb4&|JIgOU^5-N3PdnKQk}Vk!~WT5;(?PvxQV9JdFAl9g8PTkol>&dVuPE0HXF zD|De=$>M7-9c|Md4W1MvL<@h>hyZW+EGfAh_Za!jD5Xe4CvH|k1R{+|LgS5Q#PQME z1Mwin4<0?aT#kStRVnUMvQ}g}-G;$LAJ!Q@()JB=o{T350Y;f#T$ZPEQ1yH|U`&^OVEB1Y%@0L|N$w`=}^!V)*oZ zW8w}G0&=b08sOxiV{&YR*aF{`fYj)pxacFAdRKBuO4uOS3+jWKfhbMqaGzhAuqZ`qhn~}IxE2Yw$(jGTc z3L1#Z@a?%o-lefSRQjRnDw;)prSIHU^cN+C+X}XBFEwt3;OD0Sb8ra8hwGrH#|PMV zdP@ym4kjT0H@;Cl34#fqAIIQ?<$~MMB%rGj!i9q>XVvXRQW4Yzjsn`l{Wc~OBz(m}en5bs zPkl8#O1FZ^>LKp!B%JlL@4MZ|8zJwa@67SqUGH;KqIx;5FMU8m;-AN$up`NY!-fgq zuc@owoRj`(B_M$03iz6mP{YmS&L1-k@=K!v>3Jd{x%E%z_}{sa0Bj+U2v-D(Y?Una-~83ui4Z$`!$ey>$JsYAH)q zvmRTO)Ni}3nu;_Tq^gJ0>~wt#Z^iaItFrV{D$IFv#_w-3OSW#tpk4nbe;R-HMP_<_ z{M{E^aE;Th!!+o;@FF)4Q4?_&UU0Qk_akn($jZ+~6}J2+#BlkS++qwOs*m1m;VkiD z%w-mth{KIqff&TgbUOuK$^i(oE{M?Nf(RuKS@%D{XlGpz5$Zmh{OaC6&g~Gh?3U7D zo`YG}KR5?n;(o-H4}VA_DtBfHIzuk6q$NK}Skk%VM|YQ&KEGsP?ORra#V_;LmgKH; zMtx-FWv-hi>LW96J>DZ1d)C@`PgmahEZi@Vo0XfFTUfi^YQaM-A&n|=QBlI8qPv+HZ!LPcu{ou1g~JuhKDDN%GFHlm zu{c7B+nla1j@il0$$^U_t`!*}B6)J=^L^M(^q*|`19~)Q(tc2ip;h5uwL|@UoCiU? zCr{|ci1vNY<^m)L?`y5uUF6PCWbJctJjaR zf%EIaKY-zz0qL%4h|S)*`K;c5Orv+c!}1LiHFl2KR&UGb4bNqq(VK^m=CE|rP~2qe zE>0J(oqOx<-R~Eb>?zv5b6dflA}6&~S++W^GS&hl?sCAk#$n&-g)g}1tAIyV zh{mZG_(@grQrF=!u(b?iQ|hKXezGOC0DVVK3sdtq+4$gBS*Zn^`3qncVGwVr5a;60 zKC?oiNuL8Y=NyR`EnpG(?6b#?LW%g8kl$?}YiTwLGI=1l(hNhx#IBnY_b;wqy~EP0 z&~KS$X9xdmE;QTp!88*)^^W-^o1_ih33Pu7?1OVqncSid+n5;kEaR!z8s)%*%D>F< z;wbo`H}KkPue|{S9JE|0=eYUCJde8_lo4T-vsFbgo9Lc5!tA=#d=LbahArNW;(RyvXf4>KFx^PAS3Eg-HV^huV=hU3mh8pDKGG4pTAr+F{)@h*RW; z0u=q?7KXbDXREM43HM`DIEg^{FmP!&r{GfdI#@rHRk}y)hAKaO>kYbyz)p&9^6#jp z8Q-J{eEIA%wYWahKf_O(0P&q(D038kVRt6K$#?^d#xT^<=@Yt(<;L_#Z{*4pTT~R= z*&Jh~KsLfJP0rPz_76=_9nR4-;^s1>(|9mFhTErbimdfKY(OxsMaFRNLl4)?~Iu1IKRzO=3+#i2o7>2UU%v=zXSmkMayuhjNWWYZytDbt*i7T^OACMak zS0Axm6!C0uVhQH4wVTIrDgEKd<*$s^eq7(C(pRy$`g`xCE~$asuP_f|?s`#N$b-AaKp*PAzZLK0q89q{^<;RmGv*0$Z{BM}K}q04z%VP9 z=!Oby`ytA0C#Q~(rxfpW;U{wROD!czUUa`*+x?4eRlcBCXk4)!78ew?4xFVsHLO&T zfMtgIoj(Ok^oiz@gS<06lW+u4zi}?BQsnVilbgK}GYA$6V<|~n?COhMZ3USq6H}1D@ z3F8VNwjw|NEw?>)UFs!yYk?abN8bRenkHrqNAy7CSfhDJ2F7!nGe#c~X-0p^r7~*g z&|LP_404%Kc|4S9d9r^S8`XMV@Kt ztzu*F9QnDJ{YL$PkQf5E&4{e`NY+s8mJEb$gy>MhE{;u&Qy>{!X&@Mny zB6#j1#!siEW5=&T6l}dJzCjhkV72H@QB7OH4RQl7_HyPQPPVG6rd;Bjw;f}2E_$%1 z^!=@yOF?F(@8Ex$V%(R`elsUKDkVoxsRqQ#{%h}f;`7A3$gV*Y`*19LxYemTYR=Bk zR!T?OHHX(NYHV_mrj2*-VX;=HyUEMpRKV~u#d6D8@t#fxjdw6ua`0wRDI682 z$(h65rMz_zAUD!!^1Ii~BVEu~5U9^#6QBd36(k!h&Ce)BW=px54$_)z3Xv)O0**Md zIJKjK*5ni=q|tWmEQBaT5h+rK))(>}_S~E4{ThK8w8|J3<{HZnZxN;=vZ0d_G!UIg zYGv4_D2hgymt7Z<>1LarpIqhrUgSlb9!Pxj?3vx4vy`fsmXjH?1T*K=GO+X?TZPe`1HR(S6NcN=j>D4T?#OH+XDTiW!y$>T$@1{7eWXkncK& zB*?u_B5wR%GdagZs;;Y1Uaqyci(Qniug*tZ7B(duDH*hrh zR4h!41<8dQu;@rX<`<%z6=U^XUk5H1J!S_(387-`tCRIC5)mAcl`v_dx8XDeA}2{A zJwHeZgstcNSU3sBgNP2i5pk2cdRlH6$JM~?c#Z%$ddSG7VjRRGSQT-E7?YG_!Pd!9 z7IKKre8PfF3A0y9+CB*U)!A8=u}X64D>JhW_&`p_eI8Z>}{I zMOmzy!baDBSwbY1VlyCVN9{{%#6TgcOjJo!*@~E8;-m&}m|7A}Z@0u6#7qQ~B9OQF zE=)0PE($iQr6Zs!_l|h#P){<0A;vc{bwXviife$YAzvypPrEqfylF!&7>X{PGti4* zOVaUPsDfM?iyf;^)lnIyG~84SWmfLcl(it7Au1dKOPXX5#x`uU+=m)~)hxCywcXK1 zsM(^V9Zag_^HZ2Y= zXa&CYUc;@Na?!H}v}I&KXF5F!37CGBYKGo{SmPlgHbOeXp{!`WYV1tw5v~&(5*(F~ zaAU^)faFh0(~v54J-td(mpT9dw{5*&Nex2;U8$v@w~*s-g%MASd0OYb-Ztjg_355_rDv$a>e2u#b;?O^|KNtC5a5BWo$-DCy_aku+At9 zZ_NQ;oxg?>ykp^x7<%(vnIMQn(2Hmi97SljX^0EoT9Cz=kT-bBadU;1f(u+QJP!}g zGE_N=?)+f$1}i6GOkok6xMb0ef=%0sO7w#F_q@9S*SvYLC_(>5_f;8)L*P=Ig&=QD z{Cea4k&sSkoj3)}%4m2<1h&spg%b{7G$Nu4Sc}ro2a1aA)_J<#-Lq%sZqz5e4MuOd znHzwVi%2c#nWIN(ZVNr-<%dY~7hj`FL4~~^L;<@WTdQ%m1pFCY*twZ=b+_2hX~p0U zWy$A0enN*XrHnz{o#=r8r}cXbM?c1Jqcns=MO55N)Ti+v*V1^iVh5~)N*1u$i`0Yo zSs@Y-fm*228Da((^bxROp#b|>0Bq_?22|@!6Yc{Z33mmt$|zaeiSu+1%fOg)Lpct`8OFLEkO4?c(7ShUQ*?~^RV2uC4fupA&bdx;|2((WlNr5<22U~?`Cm8QGtG?krvQB2fy=s;?#ZGhIj zXkvDA>SjOdyoXo{>{$SX#xF{2fm|5mgtrp?UMh{_+=={_Pa2z!)QX()%x3j8Rx4A! zy2BVW#hLPBIRb^%%e1ghlO3Ah+cU1u;~M9wwv0s@Yz&B^N8V@3buX+2=XkWVTF5$B zPFmUl?K4L5dj5>x-Gy#gERb2jss!2Ne($IyO@tXyC~0QJmZXUT+V(!%(w0^#4Z4By zb3+7(F)61;EIN-R8$mo)q#RkxTCdNJs#k_aiN4=XNAm0Dur`wY8CFKNq{GSxred!Z zOR*JslNPPG(ph8`hZUXkVG=8upfdnWFv8^UlgMUlN*o_BHOoFltgiVkv|HP_2n3$Q9G{u@-t6)A%{z9t2isC=p|Cx-r9cnLvQ#KVT2QNnTHM`c%TjuPDNO^# z!h73tL@+>uV<7PZ5yJ-+%tpp&Oz;8{k&s}bhCtwpM+8kw#G{FEw!khynV0uZ=J%Wb zG5=$BpUc~QF6WR>zX$;6=Kq6r>Tub7c=5f+QPqfWEPELjUZ28c4LUJt(|VItfohaI zVeC*m>O7JpUy?V-80jVL7MHchk>q_aFxr$FXDyR z5-H<*f3cIx0J%kNYlSGk|8~M|y(knwm&9>Vz0DxR71X}5oeKb0&A3+G`KeL4nbyUqSQ9W_ZE5%qudYP_)(6CjR#wo{r4EnJEjTbZxG~_0>q)CN-q6pcn zq69C1`mmybMSY3q(kvS;#c^oNaPdfdt!77t__YknP@5~OU{Sa^4O6)SAJf-4G%(U( zWt^}B9oP%$r68`?jsvm~9Cvl#&+TtI-j!eFm7OpS6wlN^-JI9bmO^aiN}Pj@4YPF6 zVS2}lrd<@;L5K!d;8|?!N&F5pEL))lf-UBT412R3&!GA1@e%q@6K*v&Jn>ll7^HWX z;TGLoQKW(`r-5Izmacw=_lY)N2crqedPs?r4AMk9US(+2|Hne4d8;x_H*JwOT`eu? zz-#rPiiZlA;!*|dfx4~yap}I1n$q*UmS1=$!gJlgl1zQ#UP=q>W-!Dycj3!OTR!{0 zmmZ`njMsQJ0eQP}z*g^giJTO#p`CCHY~uFX+?IQZ4w}tYGk-cMsWD?i5iCM|w2{wQ zuUVeP>Ha9bqjxRVKxO{k?P-|p>&9(J?`n7y|1pMJk!QT7)&OSE^p8KAN&D%|xCnX2 zrMl@ci&_I|N}hV6M-E_t_Y3gr3wA%=x?+SW%1V)A+)uc NMsz%bRU7!S@-GygjW_@R diff --git a/k8s/manifest.yaml b/k8s/manifest.yaml index d50efb9..3e048b6 100644 --- a/k8s/manifest.yaml +++ b/k8s/manifest.yaml @@ -122,6 +122,10 @@ spec: secretKeyRef: name: r2-credentials key: key_id + - name: CRAWL4AI_API_URL + value: "http://crawl4ai.ai.svc.cluster.local:80" + - name: CRAWL4AI_ENABLED + value: "true" ports: - containerPort: 8000 name: links-port diff --git a/links/migrations/0035_add_webpage_content_to_page.py b/links/migrations/0035_add_webpage_content_to_page.py new file mode 100644 index 0000000..d92b7a7 --- /dev/null +++ b/links/migrations/0035_add_webpage_content_to_page.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.13 on 2025-11-04 11:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('links', '0034_post_summary'), + ] + + operations = [ + migrations.AddField( + model_name='page', + name='webpage_content', + field=models.TextField(blank=True, help_text='Markdown content extracted from Crawl4AI', verbose_name='Webpage Content (Markdown)'), + ), + ] diff --git a/links/models.py b/links/models.py index 996a6d6..872f2cf 100644 --- a/links/models.py +++ b/links/models.py @@ -176,6 +176,7 @@ class Page(models.Model): title = models.CharField(_('Title'), max_length=200, blank=True) summary = models.TextField(_('Summary'), blank=True) content = models.TextField(_('Content'), blank=True) + webpage_content = models.TextField(_('Webpage Content (Markdown)'), blank=True, help_text=_('Markdown content extracted from Crawl4AI')) created_at = models.DateTimeField(_('Created at'), default=timezone.now) updated_at = models.DateTimeField(_('Updated at'), auto_now=True) tags = models.ManyToManyField('Tag', blank=True, related_name='pages') diff --git a/links/page_views.py b/links/page_views.py index b14dcaf..7895037 100644 --- a/links/page_views.py +++ b/links/page_views.py @@ -18,12 +18,47 @@ from bs4 import BeautifulSoup from urllib.parse import urlparse, quote import re import base64 -from .tasks import capture_screenshot, process_page +from .tasks import capture_screenshot, process_page, fetch_webpage_content_from_crawl4ai from pathlib import Path from datetime import datetime from rest_framework.decorators import action from threading import Thread + +def trigger_page_processing(page): + """ + Trigger background tasks for a newly created page: + 1. Screenshot capture + 2. Crawl4AI content extraction + 3. Page metadata processing + + This is called whenever a page is created, regardless of the source (UI or API). + """ + from .tasks import capture_screenshot, fetch_webpage_content_from_crawl4ai, process_page + + # Create screenshot record and trigger capture + screenshot = Screenshot.objects.create( + page=page, + status=Screenshot.Status.PENDING + ) + + # Start screenshot capture thread + thread_screenshot = Thread(target=capture_screenshot, args=(page.id, screenshot.id)) + thread_screenshot.daemon = True + thread_screenshot.start() + + # Start Crawl4AI content extraction thread + thread_crawl = Thread(target=fetch_webpage_content_from_crawl4ai, args=(page.id,)) + thread_crawl.daemon = True + thread_crawl.start() + + # Start page metadata processing thread (if needed) + if not page.title or not page.summary: + thread_process = Thread(target=process_page, args=(page.id,)) + thread_process.daemon = True + thread_process.start() + + class PageListView(ListView): model = Page template_name = 'links/page_list.html' @@ -49,17 +84,9 @@ class PageCreateView(CreateView): response = super().form_valid(form) page = self.object - # Create a pending screenshot record - screenshot = Screenshot.objects.create( - page=page, - status=Screenshot.Status.PENDING - ) - - # Start processing the page + # Trigger all background processing tasks try: - thread = Thread(target=process_page, args=(page.id,)) - thread.daemon = True - thread.start() + trigger_page_processing(page) messages.success(self.request, _('Page created successfully and processing started.')) except Exception as e: messages.error(self.request, _('Page created but processing failed to start.')) @@ -184,18 +211,6 @@ class PageViewSet(viewsets.ModelViewSet): serializer.is_valid(raise_exception=True) self.perform_create(serializer) headers = self.get_success_headers(serializer.data) - - # Trigger screenshot capture if needed - from .tasks import capture_screenshot - # Create screenshot record first - screenshot = Screenshot.objects.create( - page=serializer.instance, - status=Screenshot.Status.PENDING - ) - thread = Thread(target=capture_screenshot, args=(serializer.instance.id, screenshot.id)) - thread.daemon = True - thread.start() - return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) def perform_create(self, serializer): @@ -205,6 +220,9 @@ class PageViewSet(viewsets.ModelViewSet): else: instance.process_status = Page.ProcessStatus.PENDING instance.save() + + # Trigger all background processing tasks + trigger_page_processing(instance) @action(detail=True, methods=['post'], url_path='take-screenshot') def take_screenshot(self, request, pk=None): diff --git a/links/serializers.py b/links/serializers.py index 59c45e1..1438b39 100644 --- a/links/serializers.py +++ b/links/serializers.py @@ -4,9 +4,9 @@ from .models import Page, Post, ImageCollection, Image, Tag class PageSerializer(serializers.ModelSerializer): class Meta: model = Page - fields = ['id', 'url', 'title', 'summary', 'content', 'screenshot_path', + fields = ['id', 'url', 'title', 'summary', 'content', 'webpage_content', 'screenshot_path', 'process_status', 'created_at', 'updated_at'] - read_only_fields = ['id', 'screenshot_path', 'process_status', + read_only_fields = ['id', 'screenshot_path', 'process_status', 'webpage_content', 'created_at', 'updated_at'] class PostSerializer(serializers.ModelSerializer): diff --git a/links/tasks.py b/links/tasks.py index 86f80b5..771637f 100644 --- a/links/tasks.py +++ b/links/tasks.py @@ -276,3 +276,63 @@ def schedule_pending_pages(): thread = Thread(target=process_page, args=(page.id,)) thread.daemon = True thread.start() + +def fetch_webpage_content_from_crawl4ai(page_id, retry_count=0): + """ + Fetch webpage content using Crawl4AI /md endpoint + """ + from .models import Page + + if not settings.CRAWL4AI_ENABLED: + logger.info("Crawl4AI is disabled, skipping webpage content extraction") + return + + try: + page = Page.objects.get(id=page_id) + logger.info(f"Fetching webpage content for page {page_id} from Crawl4AI") + + # Call Crawl4AI /md endpoint + api_url = f"{settings.CRAWL4AI_API_URL}/md" + payload = { + "url": page.url + } + + response = requests.post( + api_url, + json=payload, + timeout=60 + ) + response.raise_for_status() + + data = response.json() + + # Extract markdown content from response + markdown_content = data.get('markdown', '') + + if markdown_content: + page.webpage_content = markdown_content + page.save() + logger.info(f"Successfully fetched webpage content for page {page_id}") + else: + logger.warning(f"No markdown content returned for page {page_id}") + + except requests.RequestException as e: + logger.error(f"Failed to fetch webpage content for page {page_id}: {e}") + + # Retry logic + if retry_count < MAX_RETRIES: + retry_delay = fibonacci(retry_count + 1) + from core.scheduler import scheduler + from datetime import datetime, timedelta + run_date = datetime.now() + timedelta(seconds=retry_delay) + scheduler.add_job( + fetch_webpage_content_from_crawl4ai, + 'date', + run_date=run_date, + args=[page_id, retry_count + 1], + id=f'fetch_crawl4ai_{page_id}_retry_{retry_count + 1}', + replace_existing=True + ) + logger.info(f"Scheduled Crawl4AI retry {retry_count + 1} for page {page_id} in {retry_delay} seconds") + except Exception as e: + logger.error(f"Unexpected error fetching webpage content for page {page_id}: {e}", exc_info=True) diff --git a/links/templates/links/page_detail.html b/links/templates/links/page_detail.html index 9f9da8d..c86eb94 100644 --- a/links/templates/links/page_detail.html +++ b/links/templates/links/page_detail.html @@ -3,6 +3,13 @@ {% load markdown_extras %} {% load static %} +{% block extra_head %} + + +{% endblock %} + {% block content %}
@@ -275,6 +282,44 @@ {% endif %} {% endwith %}
+ + + {% if page.webpage_content %} +
+
+
+

{% trans "Full Webpage Content" %}

+ +
+
+
+ {{ page.webpage_content|markdown|safe }} +
+
+
+

{% trans "Click 'Expand' to view the full webpage content extracted by Crawl4AI" %}

+
+
+
+ {% endif %} diff --git a/templates/base.html b/templates/base.html index a6449db..b312bc4 100644 --- a/templates/base.html +++ b/templates/base.html @@ -72,6 +72,7 @@ {{ form.media }} {% block extra_css %}{% endblock %} + {% block extra_head %}{% endblock %}