From 2734d828f9289744ae606d3405de673a6fc7179c Mon Sep 17 00:00:00 2001 From: Wilielmus <88447902+WilliamKarolDiCioccio@users.noreply.github.com> Date: Sat, 28 Sep 2024 17:14:10 +0200 Subject: [PATCH] Use a bitmask for capabilities in OllamaModelsDB and upload SQL schema --- .../private/storage/ollama_models.dart | 44 ++++++++++++----- app/lib/frontend/pages/dashboard/models.dart | 11 +++-- database/ollama_models.sql | Bin 0 -> 70850 bytes scripts/create_ollama_models_db.py | 16 ++++--- scripts/scrape_ollama_library.py | 45 +++++++++++++----- 5 files changed, 83 insertions(+), 33 deletions(-) create mode 100644 database/ollama_models.sql diff --git a/app/lib/backend/private/storage/ollama_models.dart b/app/lib/backend/private/storage/ollama_models.dart index 3a62d3f..337d510 100644 --- a/app/lib/backend/private/storage/ollama_models.dart +++ b/app/lib/backend/private/storage/ollama_models.dart @@ -36,10 +36,7 @@ class OllamaModelsDB { name TEXT UNIQUE, description TEXT, url TEXT, - vision BOOLEAN, - tools BOOLEAN, - embedding BOOLEAN, - code BOOLEAN + capabilities INTEGER ) '''); @@ -53,9 +50,13 @@ class OllamaModelsDB { ) '''); - _db!.execute('CREATE INDEX IF NOT EXISTS idx_model_name ON models (name)'); _db!.execute( - 'CREATE INDEX IF NOT EXISTS idx_model_capabilities ON models (vision, tools, embedding, code)'); + 'CREATE INDEX IF NOT EXISTS idx_model_name ON models (name)', + ); + + _db!.execute( + 'CREATE INDEX IF NOT EXISTS idx_model_capabilities ON models (capabilities)', + ); debugPrint("Database initialized at $dbPath"); } @@ -91,23 +92,44 @@ class OllamaModelsDB { return result.isNotEmpty; } - // Check if a model supports a specific feature (vision, tools, embedding, or code) - bool doesModelSupport(String name, String feature) { + // Get the capabilities of a model by its name + List getModelCapabilities(String name) { if (_db == null) { throw Exception("Database not initialized"); } final result = _db!.select( - 'SELECT $feature FROM models WHERE name = ?', + 'SELECT capabilities FROM models WHERE name = ?', [name], ); if (result.isEmpty) { - return false; + return []; } final model = result.first; - return model[feature] == 1; + final int capabilities = model['capabilities']; + final List availableCapabilities = []; + + const int visionMask = 1 << 0; + const int toolsMask = 1 << 1; + const int embeddingMask = 1 << 2; + const int codeMask = 1 << 3; + + if ((capabilities & visionMask) != 0) { + availableCapabilities.add('vision'); + } + if ((capabilities & toolsMask) != 0) { + availableCapabilities.add('tools'); + } + if ((capabilities & embeddingMask) != 0) { + availableCapabilities.add('embedding'); + } + if ((capabilities & codeMask) != 0) { + availableCapabilities.add('code'); + } + + return availableCapabilities; } // Get the description of a model by its name diff --git a/app/lib/frontend/pages/dashboard/models.dart b/app/lib/frontend/pages/dashboard/models.dart index 509c8db..a5158e4 100644 --- a/app/lib/frontend/pages/dashboard/models.dart +++ b/app/lib/frontend/pages/dashboard/models.dart @@ -365,8 +365,11 @@ class _ModelListTileState extends State { } final tags = []; + final capabilities = db.getModelCapabilities(cleanModelName); - if (db.doesModelSupport(cleanModelName, 'vision')) { + if (capabilities.isEmpty) return const SizedBox.shrink(); + + if (capabilities.contains('vision')) { tags.add( Chip( avatar: const Icon( @@ -391,7 +394,7 @@ class _ModelListTileState extends State { ); } - if (db.doesModelSupport(cleanModelName, 'tools')) { + if (capabilities.contains('tools')) { tags.add( Chip( avatar: const Icon( @@ -416,7 +419,7 @@ class _ModelListTileState extends State { ); } - if (db.doesModelSupport(cleanModelName, 'embedding')) { + if (capabilities.contains('embedding')) { tags.add( Chip( avatar: const Icon( @@ -441,7 +444,7 @@ class _ModelListTileState extends State { ); } - if (db.doesModelSupport(cleanModelName, 'code')) { + if (capabilities.contains('code')) { tags.add( Chip( avatar: const Icon( diff --git a/database/ollama_models.sql b/database/ollama_models.sql new file mode 100644 index 0000000000000000000000000000000000000000..916d11e852a835653e4c10b8abb628546d7687cb GIT binary patch literal 70850 zcmcJY+m2jEvYyNE-3B(`M`+kED8bVj_K9rn4PQ_sMbY9xBT||f!+=1M6v?gTVVcx| z`!Vd-@M9P-`~rRq``#C}-g(pFgbLt^WP$!Rm|n^N*{)U;W+c^VO~R#GTds z_{5{~`TMJf@%bC^+nu=b{&3xmc<)xc{wV(C%IopTyXEIsG0)Q&`y|HRS$$GQ-HD%; z{=AGUcgk#xb36Wi6TfrccgkIVv-)fN{#Crr|JPUF#!L^&^_u@)ymLQZ`@_arUauan zUd0`8pSMpW^>p=X{#Ayjwj9*}Yu7UA>Qg--P^ftj%-(9G^S6@YPS^ zPT$7gA7Yl1_{(g}K9v!gn#%LjK{M#?VXW$#vQBk5>n)&sO{K3a{RaJ9r`_xUd7d0a-?S^Zj8@FG65 z7uK;8ufHz(J}AG{_w(5>*Nd3xeMsQl>K|5rj6ZLxd-^PN@Fqt8W4!hWh`X&Bi2iU_&e3p5zyyKAHAL9S3tIH#~9>(uSm`ovz)p0Ka{Pz*ZGX_kwc&rywIAB~ zyiw9qiCk{7>BA!BzlC+O_WiiGqrndTGqjQg})^*CO`RK`)`*qZPOLx^ zidm_LIR+u;{M^`A@tY^b9^M4*;l;$9p9Z+oo7iir9l#DzzNY z6SBL%`Yv>OGyXEmoA3qfnE8nj$WAD_Q|=8*aTA;>Yj+^q4G3 zes2G0^MuXvYV(S#MI)>qv;yN!3Qn2Lmm8I zSUvcH^oaa#LT+FgmWsrXF0Uk0z7?-2Iw&^d5sWMju^!pb400thJSp+*S&1ch$GT{f zJOiDWXNzInj!g8?>M-v5QMp4jXV3FJu_DF>o5d~hKYL}H4lgF-<0e+P7dZo2-q?4g zJ2NMk?ATw>7XBdCMzmP(0Wdn)m3W@8PV!Zp2L;6y-ghMdP!|7`a2R!}U4EOC*EVr+l2bUk^FbyBNh1lFEoZ}Sl5L{7rK^queGiYK8FkTdh+8eh@r*dmApjP1&h zQIPu|`~P~hrHb`!jC($V+6(q{rIA66b+^~ZE>-E!nBe8XU=jC1`XJ7VKl3Bkf##KR?Cf5AQa)(O$r?Y7Sx(o`Rsq@zjt1`@N2g0`xXh2Seqw>v zD>kvrSBT7t)?mqQSn*Qt!Sbv5r`Tnpmzd_b27fcgBTZ(mQ<_s zRZnnjRd(SzIJIS}<26Eoy<&98$r{LbKKJLuA4wJ-cs(?jb@H9)suGWX4oRXlxcTe& zgKUY=*%cx)QfEi@Oje@a&z#RY5`LVNYl*bVSd{{g=!$@)JnrFu#_F53$vI^A15O^$fpD zBz+kYc~!1xK@Du1+31qPgz%5|jd7@!N}DC_4>^1yxNY%n_#enh92(mr@t^CLvPUZ8 ziq^@#H}gagfcy);o_n|IEAnjjbeH_Pjh9bX-zw8_Zl0sdim%6tYmT-FAD6Of^WLn# z!itsasuCi3)$EIFldWOfFndNq{$_M-QuHX}^|ob|`{A*00a!3m{z>7zuv9!0CafBC zO7^^tcgA-rBO1rKdmZ!Z+A}^-YYh3Sk2=p4m7zb4D}HVC@{6qbu!)222jU@J?)D_m zAKAIrHm&G=8}E>-QK8Uosi&njw7I56&4&1eroaVdDAV|#wNh20R6EI=>c zYhVF&O?*6h<~NybV)M#3WwXYeI?soJC>zDbvW|NzUd0Y~R$s;#^kUTwgz?D=KQ4Nn zuY*=ps=M3cc)A~Zij?iK_d%Gu8I`iuI*w~QvA2rckt!FR40brB2FM!Gj;mbgK$#{{ z1MC{t(ufSxdRd~_{0`UWiFKYtu8{l3+O#I0L7J`FQ8xH0Y?*x`FDE#c=p)TNERsTc zyCKh|JSIOu-C#4)B{P(VVejlBL-yD;_tRqA7h6>;mMLq@=(8WP!6K%prIjq<**N2d z^(2OG0{#(Q>2fney_4!c6w3#(z>F{BTCl0j$}YC1Rtr?#hmWr7UP^M#bRU*0 z=?CA4Rr0yh_!(v)ESXj&##mcRc3Q!56rctOet@{hz*Q(Sjl_cUEA|<)TkC}9NvM_a zC^r-qf}xhJnYn!bw32%Ab4N}ht$Oog7HA17OPAb1V{vMt`}>eHIm=up*fCul-;pYo z_lmv~<9U}oIamR4Rrn8<;4f4X)lVbhkd0B5QN+nSY~4=HR?V%GX`99lhFo6YN>j)%v#4idFL?2RYwbqC289gPNuC>Y-={Yba#+iDavN`pvKs-Lh zj4*cfX!R<-B@4%kTYY6J!B)K}C!Xp*Bb%$=Qy;+UuEmNouDBjnN08hfzz*V-NWxEjxG!`Pug&>?T0|S)@*TIXU)eQm z5iX>i3gbPjb#z|pkmZUtRmrf(i|sKMmf#iBPatJ_ab!l>?e$JqF$10{*$ugC?-)A3 z`cH%V{aXHS<+;m}YSqHD@5&17%|;z+a&`Z;miuxv$ed}44RgxH2_3 z)l$H8u_fY>B8_^re1aV2MX)Ml&5qVK>(=VQs<|ZEdYT8(VV&=o+l@7fJ6gX&gDHP` zFkJOu@Ru6%uZDcix9~c)V$aK(b9TX9ofpIB`}?Sw8i~!e=XXASa-&UODcNi!+OqIE7F2^%{15Ump3mdR=CLpP5 zzqJH5Gd}aMT7Kl*$vXGXo{56rTq(ba-GTMW#FK-z;lXsRZ4%_kNHE)8oJb=%-Uu6` z;*qc0KBxH2lTiQVU;g*i|BP#ez1khgP9;UOx^_@YU*oTP;)8eBVM|@H*ZqE6^?iBu zZHREV&|BqM0CibZCD%Pq?W(rMT278;b^(BM??1&VOAZu!J^*lUPqbHwJx^Hmd9qf8&`P7$;9sq|a}v^!~Z})J{Cxm7iko*t|Y$`vyyi z|1gQYvs#t0fT|Cf=DZtSvk+Lzrd$#fS*``C!?He0ZNrd{kz}h5bAK=%4Oaje4~N_k z2JQ-r_6}%2+t~hLkbZi|dnufI=*cn0VCT@=@CRir6$Su_q@?63fWV^IWCb`A(zKxVUh%Sq~6IRdeL1%_H zhWNla4BDl85;6tLmA5<_^!stRylGvnl2cwtmmO}f8+Zv@Q0>~acC)tW-soj{F3Icu zF#hJw=u*WNtM$|P`2^K??}qF7MkVRp*4!&PZjwdTs8z@0d@1y5HKBqeuZa6xXu1fK_|~`}i%cw(oD!qHPL$ z!sm>1#=ply++m%W@yYsHHMlJCR&6SOB}b+HtC|O_{M8UQZ^cjRcd6ku#vY;&gQGxkYBAQhP9mz$50H+` zO0h_yA#o9=0s_Dog$g`%ZU5Mfol5Aleiq+8Q)9j|qr`M-R-B1v{t3=9*BI+C z)O8-^0o<9gI{XO~PjAsavl`EN--w>qQT_O>jQ_l`TCrKaZBH&74Lg?WXyG+@FG9xX z-GUwYEVE;@Ke4Xvm0U&^@p0MJ`g?4}8D*v7YdSe1typ&~8TeF*8zBcVay_Au-F`{J zaX!~wS6_LTxlSYItAaka9>y`-a6X@mU3Q((RM=L}ku!4@oDsf)o$`C$e_hK{_Xxzp z_PJI&JmlAKo{TA{jW)er)~&i!p6vXg&XT3c2L)MP$6m+GpJCC{=fvi{G}kl7yTgU_ zk7Bf+La%xX$1G=Bb55kvP8dH0sD9%#inKDEbuws0edWN!t)Idoun96rN%!;eYTcVo z-^g4?)~N`cmMWCRH`GD27j53RBt1Eb*3khA{yE}-@m>rmb>P@u->s}3Ur}ue|J0c_ zZ83OBT5a9yqv5&Jj1KT|Jom+*3r9?bJ;}DnGm+Pen@G<{m$ZYd#Bm{H!nvpGov4o{4F*t#uaA>gUSb z<>5&4Al9b}CsjSMYa)4=Y2j zR4?_!7$>H(gIhf$pD#$iG-uInbd zA-Sa|D#(1Ul%9`m*5{=yWkR6%I(`LuTBY{>ljRvDYC-i1El%kX<<%vWxQ z#;IrXDp%ZzS%pD<<%_~cs0?U~uVdxZh`@Myg!o|1#c#%|@5|L~{#A38)KZ%TTp8*F zu*mWA3)r5kDc3_5VD-}yVO0~MQnguIRK6aEP2hW6oe>WkLYLZ)>1w?Ds@6Yf*>4}k zrWLlzP^VqL3n}sZ;*UixAi!>{pM60v=kYU`LWS$GLhl|Vf|+4ZM}YZmegX+B6YG08 zn;H8fk}xyXG4>^+1NWcCkqgVF*Vk&Fzi<6*6|O5o)?a61Q|56mWPskWBv!6xcO?n& z0sLIpjtpee^!m}4cUB4iUWKMwuzdY}wpt%O1D^GLRw6!FrM(2KEbVMLYpCO-xhqnT zbk^q$oX?JI#g)~8Ru$P+OqC4jsvW2{1eVnb! z34zvy>$Y$xGAHb2DN|XhMKPYv>}_R@n`PXp$NK4A{5$I@Ij?k?wTR{Q)_R`nGe4c0 zZL@7ZSHAw9+bR@ShF!%msT= z)O^txkJhiO)vDCN{jD&X*C846B&=ZFlPTkzWNtxI`6MEYV>J>752(s%j}3mW-V}W- z;mgH(>eBMZ6j_^njJ-^mfglU6O{tKxB^IWw!U2dz z%8={5fRdSNtKSHn`xTB&hvej0PUJ0UH6UCi?iGuZ6hW9IszdfK9aO>%0}1kVn^(uld_2lXjH zx?{9J?P=5Zj?}h1&4n|&f>U7SoPI^_1#cn46;DtFis&aE4CYSn$dzSRK2uYSr?art zFJms%&y=fCJI4FGC!gFyc`Nx4*d;zlRg>S42hjr$^lY|5?DVu;q3#5~Q_~_swCQ=B z7TPfE>d+@vctn8l%uhUbH?6jQ86hJ7kYc>Uxz%q26hI^QHQ{y0YwTU-;~HyI zEtt1I zGJkzqzs2^526T0?b>q=g?_>V}H3>M zsGhQZ*8IH45Z=LW6Q6`Mo*y*jgE#TVQ0I<-MnrHNTYALUTgm)!xUbD%6gSzp!zv_qEfDFRvG z{ZC4_ZMlM5@2N6_bfu2(S0O_oGs$y5U;yk&Z_}DsEoZH3)|6_i*2sjBoxUFnxqT7{ z1i~mYe!con1!MF3I{E#2e1>(a79jNJb$*gOPGcPQ#D5xpz%+2{;ySMT>C!J*0j%dy z{Hq)Tgrs^7w%m!i@k6r3%*J^Y$$>z0#rXUuF~XhrLv7Oewku7@Ry`7)Y{nk-ESpEf zw;4tKlBMjt_I}$1aURQPwSihE$hlSX?#AlO6nH%>RQsESbl8A&i}t`e>Oi2I<2Kn8 zky-Ys41MaKlBz0<+BHD0h;x6C7T>}2p=dPqSxZs%DCKP2kDpgaJ)2V_-<|0FPV0kX zX5Q+D=S!24uCwc!6gllqavjbVU1wU{Mjdb&(Zwiem7@Y_7Sv9+yhIMv6FQ= zC!BW+TIF1?zEz^?n36XUqn#Z;7=Ev_xH;}%J=3Quz{|RFT-z%e183kwt^m5iA?|8f z*@wj6?iEea>zRK~F)X^~blCa$dj`bz98c$36kpED7Z$^vf^76DWg%kC@>fJW!zuAn_Umu5sL>l#K>~HJu$Fkeo4_cjsxR_GpU9I6 zvb=@Z;b%D{`8uyVXygL*s$1o#Zm!Ts--#xZiS4Yz>)82`aWM;B= z=Ln#MeU3e2WCftfJTuUBR436Hs4vSD!*(rEJ7f6#pW`}R(`KWa<lVwVU9}kXwE7F&I_rOE-crBkUnyQIkm}V-MzO} z9*&3pEF9aoCq&-c}tv9(F2Et8lvo8FkhHtd=&}MAInw0-w!(vm4>~w%s z`H_|H#&}CRPc@G?#(Fk@Y?+5xNzV*x_sqmcSUYw;-mAD&jcY@ZC7HMN9FOA!Q1X`S z*1U>UfC=iWIE!JG%Ai&GAL9fVnV&&2oSrr9ARz0y4~10ScR*WW)K`KTepP-$#_C#p z)rh{ER@deY*M_{0m4K#XSFl@WD|f2b@%DAk?#yka#swtGE;;gWAx~C8c(zvOk-3(M zKcrR{B8a7+@bDPF}rHQZ8p3tsa7p0S5%aR$HNe)`A+Rub&%{} z$mZ@AX2Iu_RVJn^J!672S%v;Z>w40FtU~+fR5=1K&d*oij3C$i`%beW4YT6z7#|K! z)>1!j%F|qQ@5Hg06W(}Ut|$rV-l?4V?qwC&AJ|ER)lb-0Yc*_HL#^6(N0^wLGuQ{e zC@XXQ!}-lr=NW~KU&z5bB#{QAr;kl{RALo#rp_238ye#$7()Hb&3Me7y%yc?R2r?4 zR_1A0xUV8FdoZkW{$$a3m8x1`rJJKW_K!r^q0R_ko}aCmYEKbI);iZ!?LTekwY2h9 znJUA?E`KWw89b*t@p?e7`)BXKo&BDxMqrlCP!k?04%g>lW6zn_*LLnIAy0Tx`EbQt zxR31^6LR-joBYu45J5q5jOmF}s*Y3tuUXBbf^BtXGR|to^}E{o*5@+duCN%X zGaC|=_2x5R@TI()R*7DG&8<>azt!BF9Ypf{+p$vWj<9gQJC1#6S+)Jm@iP6ob)TN^ z!*|)nldlbVq0ig=5No9Zq}sYWrOTOMy{2)DsAZA-Y|5MMrSkOU6Hsd|-DXdlR@iDk z>z>kl6Z;OS#jrnxCk)itGj}6u&g%;)J0fpJ25gb>@IUX*!a8*Z4I`=Fg*3f8WqJn3 z_$f9pU!OtW8O^70PCe=M&+b}}UZ?lU|1s`wB? zl(KkN2iOnu-&X(X-{=|d@fCGETiPkJZih>qCt(S5)J13P(IKm(Yl$vlwP4bdcnz&C zMNP9Y=}Ep$jr(Ez*)6^6`coi3HU;_wS4k=thT?;bh-Ljidon_m}^^j4@hkA=;MSQVW zCjuaOtehOkdP3s#$cVwnSo$H?QZJe;ih2YXpk9#jV!lb8IA;0lRec7||1w7O9Jy-> z9WpZa*6&*zl%sEwRU0Cs6=lx0B{ElbXSj!iCUTYIZkeUv&(1xm_DwWrmLNW)x-jU< z4kC~CJ|r(6;Y1=N;CFJK)O@S0d(82=+il$$X5@#zG=3t%qWcK<`J||wLDhIMt)eA1$stM{3!c+OOudT=_uAE@8NrWv z^a7ixgdW8XD8-LH=SIw>(Jp8AO36Nd3OxKFBmu(myq4J9`Y4X|V@KkFoT{d$scQKV znbZ|*5>H%?rg7yHqesw=y(Fu?020V;wi3O@%w{fbc!))_4wqu8KSv=RBqVb9B zlH{MO_orvC+lsR?Ft_0`HgC$rToWBU3TwxO9h{k_~}es4hv7ru=ir|xueAP=KjXI@p$@+(UzkmkpK^2 zFDaQe`{9Mcvj4F*6ZJt+a-Ng8qc)3QFRONKXb&>XnM&1fXH|RQ-P@4%GV>s2fEn7A zZ3%-Tx}LSyl7Ry*@9s*8CEHoI6TVH3<5jlHPB}GaEx<2Sv6lslMT3TUmO?F!b+$$v z12?g-v@)@>W$9&wLs;SO22AWm53*sn9nrD%`Mmjx+hwUV*STq@=4@1nk-E54rGjijE^{w!pC zGd|_|hO6hS2d#T{M@lo7<(gfay()S9WU zF?!V3Qhnuh?@iW-ZTgx@wCzkNH03U)qGe{CW`278wivD=fXu-$mk{5jr&EmNn?U^o>1j^7Va=<hQDijtL?!?2cL;Xp|=jEc4`>p(WK9wWGHEJ3YzpeE;n-ai5Yi0Nnwk@0yJ5 zAb7gIX}PwtW!?pkl3h}(p^nMwkczh;Qk+_Tb8 z#xVV?R!yZ}5osJd=hj!}C1Ad^_^PmQ^v7S-{xkp6S+~2f3z1Ca{5x1zMQyY9^QG6; zsUr$1b7V)7cI1Gy*t|U8;o0jutFO!J`k($Vr@{R?#6%&`(y5=$ix}PfLA!oxZ~0_k zi*Uu%!hYBYcyj)Z+9if5e?x|NF1Dgqh~RKh?nckBpzUvm6@T0g^kwyeN;Db@nm1Un#3wBr2oowsu7{lF3Af`!h{VBmWS98a+!_Tp&2Dc2JN9Zziq zs-fI0qx}>;07m^Oa;NVK9v~N6w7fd^-jZeNeWL7`MP`1kN{1z0e^2Z9g!PQcx|g?; zd*K-e--%=O{K^}lIaW$Xh!xCt4YWlG#$GS=)=mT%jBedKphr*E0q(G)&bg$cv7XU% z^bnr#FC9d>I?4>QHjZ4z_1Rj##8qr&9cxOw&|U$$VxSbyK*E?9-Fg!^+8;k)#HsA=NM%1=L!-|~LQ*YL7-^r}bIeoj@b>QyGn--s;pR@6@(#lJjr zeLeEdyX9x%gL)`vAN%`c_4lj4i~pDZ@O_q?RB^wIMJ4B(`2Fkn_fAC6>+%0ztgMHId^9G5a)Rz zrHfquMXV0l-ifN#{j-&S67!pRXa(>&uKObX@;~yp6)0MB_IDwBR{QCqY(8E6V|g7} zJ}jd=icd=u58|h}Ccg{$-wh2g59gS0vJ(il7ZxZl;H;#BxO2YEp>=%FQR5$+jnA&t z%bmu@zlFfZXX78n_*XlPPklz?tM@1g@vZfJAa2(ksYd`q&gF z|5_7Q!pQi&S!slooNe-#YL@cm6m}(^kCK z(T4Fu;V+!?a_Gpj9Ce%pFA)Y`s%)!goH~em&pD8q*MZOX(#b)rB4;_Mq}4B09Rwya zhfqG_NMC8m<2A>2xHQsX9gZc|2Y%QqTHzHr$9DV>WM>XVtT`v;oZHVkUJiZs>It;nzEcm8pL4tiE z*5Tmsvam5f?WOg@&~2`RD9qkn<HToar*q*dhpHqddW?%gn@lo-3 zuVh;q!Q;sp$+vz9?So&!;{8B_!%ir9R3g3duI=S@lvqTc76h{)HC$UjjjydZ(iY5z`=Xv z!&e61*BoT{o`w2wFZP1Z(R0v#mh==>(r*r8u=4vlO7@)tkM|s~qwMnatAFX+f>cES z^{o+B(kt&KMzIn_;yOz9>&59iYVbdNB&)0VdaoQ3bYdNvW85$64iP9SKmqCIkv?kk4uF2EVcFai;Bm^H&mahy^soVFY7%ncB}f- zwlM2>@N}`1#4D;4y(0T@iR`L>)z%A_=v8CUxznB_trt}6RrS(I%${Q!4SV_F@!;#p zw^A$cyUK%OTYRsds0}*5=-e7 zOLPvJ>bErp_hb;%oZ;}EWz0=`jyK7{9h4& zdlDl)9+tUOEv%zhzx-6aCuUMSW)|1i=(qH#t}%;HAfvu#q59Hq7EnMPiJB$BEBj#$ z(TiCW`QgQ5&ewB2V1W7(;LW(R)^~3tIN+JaR^6*kfenBIM?>t~PBo8NzyZ(lHclkq zDFvRsRo%EJLt{3O;HaQ=9jnn+zbLAX z10jN=X|lt9wM*6_L{RjRXPO_v=MTC?AayF#g~P7X4#;zQRUH_YwWtd>_0oQ`sLJnI zh}4!Em5n}mG|`kus!pG>b#sHB3`F&D&qD0+EIk>Bs@I-{=y3UL9x(4(H%n8+G-smH zlV>JjOo zeOZBOY3eT574*I_N0lzmQcJ97G*Ep=9c&oec4`Hx5M2rV!`MXdJDHAu^|Y;!g60rlkT)%5lm`MAAar zabW}azWio8j8K{G)iPIY0e!twIjCpZo;@r3qzkCJB-x0W8XD`x)Rcc}zk#y0%nM-E z{qh3kpW1c6Of~1AyVHwtCZjSLB==(o zif{g2`D6!RXg_wKnB!S${q)QewO7!yFmiHUs>(g31~>3`$8XcFVFuc>B>t8wVx9f^ zeL`x_G)A3%^+}!E7sS;qT5daeatpgm=m;w-4~X}9Y4^~D}}sdkucFVe|#w8w0lvw#OVOLD}XaZWaH zt;8VZa^@gC_kK}ZCrh{%K1mk5t&=mDhdldQprx|y%(Z*3M~I-TXM0gWn$f5m! z{dSjPZHW~9s*Gq!NTD5fplr_5FB4}T;e__uZR?2+<`Fh%x80>!O}*oi9z?Is19KT2 zj6?NYJ1mu$@qKlt-$w@j rIDY3m=?CR}=z0~Lwe9ymZuE8Ewz6)vb=T+Y==ENlD*b3UZTkNMF0Bzl literal 0 HcmV?d00001 diff --git a/scripts/create_ollama_models_db.py b/scripts/create_ollama_models_db.py index 8737a8f..236875f 100644 --- a/scripts/create_ollama_models_db.py +++ b/scripts/create_ollama_models_db.py @@ -6,17 +6,17 @@ def create_database_schema(): conn = sqlite3.connect('ollama_models.db') c = conn.cursor() - # Create table for models with a 'url' column + # Create table for models + # Even doe using an integer bitmask to store capabilities is a waste of local storage + # as SQLite INTEGER takes 64-bit while four bools take 8-bits each for a total of 32-bits, + # it saves space in the cloud storage and makes querying faster that uses PostgreSQL c.execute(''' CREATE TABLE IF NOT EXISTS models ( id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT UNIQUE, description TEXT, url TEXT, - vision BOOLEAN, - tools BOOLEAN, - embedding BOOLEAN, - code BOOLEAN + capabilities INTEGER ) ''') @@ -31,9 +31,11 @@ def create_database_schema(): ) ''') - # Create indexes on model attributes for fast querying + # Create an index on the model name for fast querying c.execute('CREATE INDEX IF NOT EXISTS idx_model_name ON models (name)') - c.execute('CREATE INDEX IF NOT EXISTS idx_model_capabilities ON models (vision, tools, embedding, code)') + + # Create an index on the model capabilities for fast querying + c.execute('CREATE INDEX IF NOT EXISTS idx_model_capabilities ON models (capabilities)') # Commit the transaction and close the connection conn.commit() diff --git a/scripts/scrape_ollama_library.py b/scripts/scrape_ollama_library.py index 1e3cc91..e209899 100644 --- a/scripts/scrape_ollama_library.py +++ b/scripts/scrape_ollama_library.py @@ -4,6 +4,19 @@ from datetime import datetime +def calculate_capabilities(vision=False, tools=False, embedding=False, code=False): + capabilities = 0 + if vision: + capabilities |= 1 # 1st bit for vision + if tools: + capabilities |= 2 # 2nd bit for tools + if embedding: + capabilities |= 4 # 3rd bit for embedding + if code: + capabilities |= 8 # 4th bit for code + return capabilities + + def get_soup(url): response = requests.get(url) if response.status_code == 200: # Check if the request was successful @@ -92,17 +105,17 @@ def save_data_to_sqlite(models_info): conn = sqlite3.connect('ollama_models.db') c = conn.cursor() - # Create table for models with a 'url' column + # Create table for models + # Even doe using an integer bitmask to store capabilities is a waste of local storage + # as SQLite INTEGER takes 64-bit while four bools take 8-bits each for a total of 32-bits, + # it saves space in the cloud storage and makes querying faster that uses PostgreSQL c.execute(''' CREATE TABLE IF NOT EXISTS models ( id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT UNIQUE, description TEXT, - url TEXT, -- Include 'url' column here - vision BOOLEAN, - tools BOOLEAN, - embedding BOOLEAN, - code BOOLEAN + url TEXT, + capabilities INTEGER ) ''') @@ -117,17 +130,27 @@ def save_data_to_sqlite(models_info): ) ''') - # Create indexes on model attributes for fast querying + # Create an index on the model name for fast querying c.execute('CREATE INDEX IF NOT EXISTS idx_model_name ON models (name)') - c.execute('CREATE INDEX IF NOT EXISTS idx_model_capabilities ON models (vision, tools, embedding, code)') + + # Create an index on the model capabilities for fast querying + c.execute('CREATE INDEX IF NOT EXISTS idx_model_capabilities ON models (capabilities)') # Insert data into the tables for model_name, details in models_info.items(): + # Calculate the capabilities bitmask + capabilities_value = calculate_capabilities( + vision=details['vision'], + tools=details['tools'], + embedding=details['embedding'], + code=details['code'] + ) + # Insert the model details into the models table c.execute(''' - INSERT OR IGNORE INTO models (name, description, url, vision, tools, embedding, code) - VALUES (?, ?, ?, ?, ?, ?, ?) - ''', (model_name, details['description'], details['url'], details['vision'], details['tools'], details['embedding'], details['code'])) + INSERT OR IGNORE INTO models (name, description, url, capabilities) + VALUES (?, ?, ?, ?) + ''', (model_name, details['description'], details['url'], capabilities_value)) # Get the last inserted model ID (or fetch the existing one) c.execute('SELECT id FROM models WHERE name = ?', (model_name,))