From d34414359cae804b0b10fa7c0d25c2202f62b1b0 Mon Sep 17 00:00:00 2001 From: Andy <88590076+AAndyProgram@users.noreply.github.com> Date: Fri, 9 Jun 2023 21:44:00 +0300 Subject: [PATCH] 2023.6.9.0 YT.MediaItem: fixed opening paths to downloaded playlists and channels API.InternalSettingsForm: add members distinct API.Mastodon: create personal EditorExchangeOptions class with some Twitter members disabled API.Twitter: add 'DownloadModels'; update algo Make progress more informative --- Changelog.md | 13 +- ProgramScreenshots/SettingsTwitterUser.png | Bin 7121 -> 10176 bytes SCrawler.YouTube/Downloader/MediaItem.vb | 2 +- SCrawler.YouTube/My Project/AssemblyInfo.vb | 4 +- .../My Project/AssemblyInfo.vb | 4 +- .../API/BaseObjects/InternalSettingsForm.vb | 3 +- .../API/Mastodon/EditorExchangeOptions.vb | 22 ++++ SCrawler/API/Mastodon/SiteSettings.vb | 6 +- SCrawler/API/Twitter/EditorExchangeOptions.vb | 21 ++++ SCrawler/API/Twitter/UserData.vb | 115 ++++++++++++++---- SCrawler/Download/DownloadProgress.vb | 20 +-- SCrawler/MainFrame.Designer.vb | 26 ++-- SCrawler/MainFrame.vb | 2 +- SCrawler/My Project/AssemblyInfo.vb | 4 +- SCrawler/MyProgressExt.vb | 59 +++++---- .../Attributes/Attributes.vb | 2 +- SCrawler/SCrawler.vbproj | 1 + 17 files changed, 224 insertions(+), 80 deletions(-) create mode 100644 SCrawler/API/Mastodon/EditorExchangeOptions.vb diff --git a/Changelog.md b/Changelog.md index 7f9caee..3d97a09 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,12 +1,21 @@ +# 2023.6.9.0 + +*2023-06-09* + +- Fixed + - YouTube: opening paths to downloaded playlists and channels + - Twitter: make the algorithm faster + - Make progress more informative + # 2023.6.8.0 *2023-06-08* - Added - - YouTube: append artist name to music playlist output path + - YouTube: append artist name to music playlist output path - YouTube: save thumbnail path for playlist and channel - Fixed - - YouTube: opening paths to downloaded playlists and channels + - YouTube: opening paths to downloaded playlists and channels - Twitter: profile not fully downloaded - Corrected form size for small monitors (Issue #136) diff --git a/ProgramScreenshots/SettingsTwitterUser.png b/ProgramScreenshots/SettingsTwitterUser.png index d47197fcd6e871e6091c7054a0cf174b48c0e99e..af6ec99d27110ed7d31695d86406ffef6366f3e7 100644 GIT binary patch literal 10176 zcmch7cTkgSpLS4>f(-)#B27b6s(^w>Qvs2V^d1EP=>k#$1T3hb2qF?dnuT747$B6y zf}n(=v=BlOFqDK4dJBPla`fzb&b+((&CYzYe`FZ)Ja_rs_jO&rmRK_ry+iwt?+1ZE zhxD)Nn1euU_oDN;3-W>e1MZ#?f-@ znzA`%qtkW)rnsMhM0NS&_<=3EAA|tb-8!Ok5Cr;m0&xrkdU@%H0tj^R!3h`O&ApeJ zftL;`UISO!RogPRRnv-7H0!L(HG}o7qWbCB^8UlX^Zi$N7x;0}_b<$JRj~JS;CH4? zdo!`~uQ&T}sxIgKKHXS>YHh1fa5vR@OH%O(CwXQxlhh6@aZoyt}irw1dS-5LKYHHcd+@7z4TO%jH949Bysvwq|_tTj{Qz z-hL0g)P>KKOU(yBASDU0p*P8I&r~C9%2XYim~o`OO207KQ$xq3x97{QWA9(C=d4)K z@p7*r^-V=Zq+bUE$9ndG^vIi3)}XHSrqB|V;jlBONXJ@z!>qDl-D>wY4qh9Jds%+n zkXVsgqk9zPb~uFLY(3zEdVC z4a$C#gxn1R9lSmX(WHisc+GK$tbG>Q^O+h%;in&(EI#IO#qsNE2PJ3?^DGf@{%OSW zMB>;zz!cjfwjB6|1|cssM3s(AR@jHB``s(B)}8uuVUgL_=D>{ZWe+?zZvb(XTOb#y z8niP20$u;W%|-v-7Uly!VA!Qe*wPj{$#wu2K>IYA6F5DfIplW3Lc;uxN8ZTxMg0cG z=YZ6!dYUGy2XtE66WBoS-Jae~y3#TcyWR}7DU^kl@J281^ltOWrcG!q9a_0d{d0!wK;NbMpkKSm+nNxxf=^u;f=q ztAfDAGboQJEjKRa_~rC>O6q5a%mu)w%1Q1P7yLnY!N3GAN5OPbt*NX-jnOb{D*Kb} ztU-BZsZ7SR=LNDZ$+w@CG?JvL?!}ZmSQR}!jLVe`lzs$Jr+bjQDs^5@a}m7qMud4w z9o*U`@gZ+<@6w$WK7@{2VhrPUQf9yEE|8!&>t(Jg`5UoSx{>E?JoUO{@SB!hPdgUN z+m&ewH6y;>wZ@&6Zl30HjJCi~SeEEVY~y8LPmFq}ckZpRkS&DH#2#Fc8esI1OHV^| zm72v=m;ISZwev^L1cHm%ZQPTqCbrftIulLYTBrth%ovy4qF_D^ua89Ot>wC7<_P7x zLFsaU!3bM%NB&kZ8HE~zZw7Q~PGBbPtQ$1m0p}6!PO9A_)W+8#4F$a$pc#eVVg!!< zXzv~lto8LF=?U8^;VJD1w(B7$#(8h(%G_`hOnKXE3z5rxAvf1?*#0uF=W0#`G9!Q{ z94PC7fu^=D@+t*LTP!sPl{ICKx**@XNkHeQLCFeN(h3-X&!KzT7r<4!K}H;ad5+*D zE^S?(w0nm-n}(~gYi6m^a?-7V(7CVn`iy5Nh969<@(UrcaMcVc=#57&GsNwMf zU-=%$RIc$4g%quHeX<9g(LNFopI*cQyvbdJLkuF=@iGh`_1xY64&(psPiD9Q=>WC! zy8j9Ed;b{d2i|k(Nb{`dGE7|`Sm|lqa!krs&Z1d^GX5cA`L+2p;>Is+oClJKuQ;?Bb|J?97%#@c~ zwLIA`;}3AHAcv08kv$5T$0S^mY4>*T>pdXf_%bJY8)hNFHG+%AovwA0v)~R;F}3QR zDKgQ2=3e4WZOAZq76`4$4k_^Mr{PsRZnw@oEsW)lctOBRLt1t5dOEwn++ z!O=e@MVnqKPlYuv3Quep{@(rxFo`^6D2LIJb*i^4hvJ-R!wojRBaL?D&Q12&u*nRs zt{e@@j}H&f69p&6E83p@;vqRG0rEoi)k0$6;VR|Uux&0x`U!RyWd)xrGWagFkXf_B z+^BkcOkjTf^y~P9yQUf|mP!`L@EzxcpStQ5&mpu1yQ*H&jF{9R8LPFvO!Z=el)zXA z_n$RQ6+!q?c@8Iy6g2snWG$?1p%+QW$9}OAsce7bn^|pIspT-4gDN*&PD7f!Y&>45 zYLc5~w>g&^{4a|RK;Ab~? zqICYIhD-7jRIfJ$&h`fdA1ZAp>V-Nw*YT|so8Q=z(s(u1eh7|n@GDewMYj?ow*y8L(BTbq z++&Kj>(RXlsDEAkfSfUX5N%j)hiYk$={ajvYF(~~a?KFO`@l90d$!v<$M>JAp}p}a zw!5icQ#sP*x<=Nx6InJ|GSvB$mO%VEB{GHFSlJGEKxyUoi2ZBNTvd#>{ih4FUiX(f z6o$}8%KaJ_t_0!29&vhmZWR`ERQs#GIm<|Ny#M;N_^m|FSPe6IrPk^0cMW@d8B2I( zz9%tsDb?`HtA+lyw#GMd#)eG}b4g0QxexwYAQK#)pyD@FR4rrCrM_fzZZ}f0s1h@I z9Ho(&EAQ!r6GiSz8WOj*kpq86tj{jI8s%YuB6jI*N{WS;L+aX3duamKsz`sy%1Ogs zbK&Ea(1JBa%UDgaZ>CMxbOiZcz@yEe-{g$F1mT^f$LM~kHuA;aggzZT%4jt*ZdUR@ zyq%Q(3#UfS_YnlX3Z8ck?tt>^{ifCx@Iqa?2w2`s?1%y&0zSLI+9JP4+(W$Ll4vW8 z6PVB8YkE50b=1Q*5|Qf0MxAFl39V0y0DWp)cSLtA%*=UrS!D;+60_?OG-Z|*Xa7g> zdid|L!GND>mSet<#$lo`N`%9vnIPv}w%5c9%(|Xyt^W)0;(d!~kTT>-Rppv7 z3%IJ-)au1^LxN&Y2)?5m)s98kXO>B$#sYC=D$HqOcq@(6nXV!B8Zr{%mtn*GtkTB@ zeqgF3kS+5B`+fH26X5@cHtmAc0E#W0_cZd!K_^d`i;SD6$H1^rybZF>7?T5=3e+NI zJ|t~LX^}&6+Thbeg4_+EG75LD2xk7FBnk-7G3iIKwnxS^x7R1UKF9G;ex{fzqi$q~ zw8Mj{)c=eT7xrfN0Zf3}$&N_iHdb9CLr@nxqnugbcVvy-fW%R0jv!633a6@oY zWBN!zXR08}BaElJK1B>aMw_UTkD~pCFQ6mV{DLQ*sJvi&RxjiE80|GUQu#zb;O>zF zo>!0sr4u^aLaBqf8pQhRTA%qIo~6IYRsI|$C4P;&DshmGKtHb>Zb@J{8ZMTjQ5>lY zc+*yGq_Nbl)OXM}=OuRXg?GEM7Id+QnP@AV6DP|}h&leg*E_(P(?vItn@AAuUmSc+ z!th{JO|U0oa_5orx`3a)WF!f|@-2>=_at5sEt>+r6lF0hg8HGWuJ9-}+~BH_bu-yK zL`40XF@WHI2uF&`2+PX*h|i}W!hAYIf&T}(@%LE&e~jBxcmgWAO-dGDxw%53OECcR z&xW=E9JrIg6zHrm)_vXGNPU+U&`U*Cup&$-5F36!4wj59cwQmP{i+^P(KmIaSKr-D z{a!%({vAgUsAUhh7O{#rZ(&DicUiI&FP4yS!5ze~z2CJ9cwiS-mnq)l=Bj4Vci+e@ zme}ssGap!$TJ)t>U5@Yc)F&Nkf*vIpr%$x;FPMGl@x9M>DBssV!Ai3!n-XwzWBK5a zPg*N6eCPb7&QHrHuQIkHLkhIo&IPqvBz*Yl&DeAIk3WE91jSt?uS-NS8arhOU&PLM zqsdlH>^;LtLe!t~)F6}-d#zl_cutUMh@!JIFUV0M)dCbMZ-qn~Nu^=?%i5w`Q!&B?Kb3r~A8$(aLO<14mgF(dzl-pdd?^ zXMz{f?2vL~SGNg2o0t}h6T3<~;vTf_YokS#(aqp5jY}%EgFB-bUTT3Imtn5t5D^gX zQ9#!L@IVfm=E3j3HLSCLYR(6CJ0$rST!bjL@01OC+{`T{?P=0XHX?O8YSS2rD;0L9 z{L6;scE7*RI@7OlwsXU)NaH^+l@2mrk-HS1)qIq5kDP1jdZ|7XNUT|0#d|}fa$eNn zyFxf>Z&4FJk0*VuIgQsg{O}4(t7)VLtjZhnp_ZL7Sj`$Z6w5~N`4;A-+6D*NngQ3X z;0YGEF->$>2og0OOu2WpRZiCibq$=uYgnbD_VY&hC33ZITKJ((rnxpj{fAW_26ZD| zjdv|9yFukjA?^}s(rIcwsi(K%fgd+t!0d}d{HC}5vf8=%`o$Pl(TA0`w2D99rw(MV z|IAVUJ%0aH5Bim$=9#jB_%jaaeAP+}o^%~SZ+flsP~IM4Rc%X@9-S>^|TBX&(sdN$;rx$5ONeD-JPJ5^dh zYj|YQVURr=FjWul!q_K_fm(5!x7$*T=Yi7G$q<*j{SOch%|;&0^_s?Q;#4s!JA!aZ z$ys9~*fX+}5~(_GxO}m`mB*a#gQRX^wXw@X5lXAz#1XGIt%CtAUW;EN?D{+66kb$) zJzqKz+~n&q5{zAp^fFvRYX&#b@Si-;Pcs9fNr;z69`HQgm^TeV zWfwyHaO&MjJ~qvyqpG^q`4>wsc)0tw*rna+>+FTSR`R7xfko1SDpup~jn?$1NnXM1 z^=neCX!mwIZC*nlPA)H5Ge1{Y{RkgW2YoxwS~d7enby4bniy>HF)sLF`slo!gt|B&Heht?T2|)GYbuuzD}>ZfkIvKHOgjh5zw>p zPwQM+h{9D56QZtShB;FYsfF0W%A9mnl~z4w{33|LcQl5n;F5iJudp~xfm9&R#PeJF z!_F?g8m)B_=CgfK!w8h`le=bvZ-~-zV|?)4Uu6izAmhK<^dXfTKF~OQL+N3i>eai%|{+t(B0ou;2_@{`w|$cAX?rflXM)2P{IF;DiWN8q_;idD^5A@7fhQ1Dqo=0#-ZZ@R~itX`chSo{mW*bmH4hr*Zl~ilrS0lTyBP`Okk0RuX^T=&ySW~MNwjJ)O>eQK z1x%Hs z>1qZ$$9Oj#6zoxRvLq5>4L?MF@deGVQb`&?QfXLkSJLi_R2GOSki+`#{k`*~U7j*s z`qO6Y+()_YoPwUMjAT1Jk75FVdj=Srhb*RZx?V8~2KRXDfkLe7n-LFMfzKN;(m! zH+kCY+bo`<&SD*eze?Y)nphCYDrrM6dvEwQk(8^2)YEMJnE841S5hiage}}x@Lprh z6RpEjb<=3JyD*mAbMpY@qjrv805<%MX5g!Twk!Ut`sttjjsH(<8S1cP6@^&g8H+7f zq51{&uj&KREpv{13e=-C*u8#c1g-tCQdTRZlOr)%M@a{FqA6--`#>}{01mP71I(B7 zNh!k%Jsa0WE=h?h6sw;ESkOz*acsz$NH}NXDQSiAAx&I4FAUz@&dH)!3BuJa<+?en z@SNri+c{f!4lil|8?ILYcvWGtqwnae3!qzvSfcOLJkl=Hr*sq5{^C3eZ6H^1tIan> z$;uAXZZF$a4}ZA0-(IRI-^s&{;F&9S!Qaf*{;FH9eu2lTJIT%~X@wt8LwD8Jv7(eBJqw5fw=ldt95J>Rm@$#Ft`bsD=|87n%dSdR4Jn)VfYe=UMay2 zvd6*N#_tyA_P>=7osmWoR-s(p^6*VpIY!yd89&ZhB_v5zFB95U8tH~fnj!7)0~ZLT z%t$Kos)RzLlNSb}*j}!%Tnvi83OY4ARgB0Bt2>3igF)!Ts7-4Y|0a!rbn+ojk>3~( zedu*0Ih+RHihQ+r3BKt&wDv8mEdBR9!C(d#JNjHVk7pZ}w>GM4xxK=lMuD~_v+^s28@5pd{C ztsl=RfZG$cW?P1+2g%EMTpbSW%`}5jaaHB&jf&(fMkYAZZPi zPD=Rf+e3ifRF@C|weR~SoK0jCX?BOvh>KsCKWY_EuaQSeQs2pQG|-WHi*b>(u&sTC zC4sU`O%52wmGmJfmfR0D*c46-TzZJ;V~}4})%nhbl*^nE&g2By|LO$%pAh8l8P4B3 zhXL`n?{0k`h*<=h8AHISv4g4SSw-GSO%^sSn}?0WvlA(nVp1IqV>PXD%c(a@1WE)R z|K%_;#ihQsQ74^y#ID&n`>QGJO+YIPgKqutD~6hT3{$bO>(1zWyng2EleWCYm}5va z^HFYSg*nqrf_i}lM1f7K+&A9af)y?@SV2VzBr(n1Woy)!+0Mh7fBX4sy!kH`UCj*^Mvf z`Uw5P-d)8$J)zmdnrtu6u@DXi#~a;*p`RO^R+3mldIyWi0c!1l&0J_+os(qt~$ zs{Ou8t7fuA&*!F=fxwF1*o)z&Hu(0s5KdJwM2R}G_9@Z{OLL!7tRjqb$k)O5*1E!5 z0@KKSfp4P^yWxa&N1WmCVzsC8A)V-`E!XnKrmBPd0-$!^UxX^n?{>lT8hq_v6Fw8D z^|Sna8yuVnP-@RFN+D+dlTu@0zbO?W^7Qz`4cVpqymk+4mgU;R-5Ymc^I6QOAy-=T z8?5rDmc5`LewJ60VzvFafvY>|B9vwZ^pVrm{@y14Z~e|c(xJcg)&H4CNr4|59Rf0{ zl&L2wBSoxITxnobmth=nVr`nWg@Hyq>*fv-*=R4KB z$HONQ)q2OIE6ggRC>E5JU}`URsmq43O{whNX<*FRH`L6xNmsJ^lYh?c>>x5!G6$O$ zMl&Nwea55Muf-W7c!Qbk+D2UsIMIzlG?cg6x4kIex1AL2sImIhv_xby$h2>=54V0G z_$Q^bk80nnx4qTX=Pk1Rc5{wHvPeT^*b?o@EstG?03mRCQr&BT6*Re6_R4!j0W7nj zR_)vfb*!{+BV57@vsK}cJ!nj!EXmstMBXuKOQ=f2exC{gaXW7L?`*p6;C8mxN~h!+ z&4lWHa;>~Z(y&QUmKv|zB7Y(sdP~!w6^Fg`MZn7m8;ChZj@J6_U}o-iarh#AyF#K) zII|bt8rnN(@^xXVIPE^I;DZ%qvU7ogHGWlpA1KR;{yu|i&ZHgRhRqh&Q8P7R;*Ejr z37Sdyk|Ng}pRZ|4FxY?aM|?f79WkX*J+#kS6CFDD%$9HCk}8-v}!NiGtODf8{X z(m-SBI18eBj^7VO_`>eQgWt}tt?e{fOf-3Y42D8(sU48hjZDi?^&n7RIF}CK(^AyK zR@NmoUmY&9&v&+KvTtgAu4KLu@|lnNbaCpJEiM86Hz+W49B7dyUD=b|wV@TFXA zafoJ>1)O=UR$`tDypK%je6HL4xoHBM5%d#a7`1lj^(s(eKD@nbTB@zJkwVFT#>6cf zQVLL==A-twsQHv`NJ}z(U&F`ezOok5lchRetjiw8P+e+T&wuP|J?i@?emzz)=T8oU zZSE}AuMp>lEceJD5*}|mdS>x2eCqB;9*TOnXJzvbzM=0 zK}J6<;f$!9n72n&1;Pn-fl-x(Cf<$(ZvX}@ei8j`Q0EAP*}sO%uVG$yu)+q@`@Qrx z4M(@lt%ReWgeKL6y_FW5+r;zq;6|ewF1!|xoKB0zNRH()?N&ZkGG8Rmr)Vv3Q*Ly; ziRUdY)LhEIj`FV^3rwn$u=0#30}_nrI)q60^hqEkJtGpJo2G@d!FRb&M!ZemXv3=K zi9<#f=xS~K+RpGmS>$V1Q{&8iHIIg*Lq3a{>WXY(a;f^ZU2(aNXoF!`zn#~pKNGo` zgPgBDJ9^|Kj_TyqWo7eu=Z}iO7J*1Fd&+D#8a%@e5w1I{@$p&k-F?Dgpx+Ss>P=m8 zcoI_{<{$A>c`5V8nBmUZHlG0)jP+BGr&NftxWJtTHpLqAaAvn*-z(@w>Wi3AS-aQ$ zxj~788^o@X!iu`f8jGv7rIp?sqvgbcjm}ufl3{N`NpxaI!jqfX;u92X)zY`V1?j$y zH|>a0rZaZ_ll=>DS^*um6ZPAA4`?8#t&Yr2U_T)QZalp4KKsGWSR#K@+4HFRdz?jS z(PMo3IjD9EIen#lJAGVH(=7rW59~hX1XC^7eEpTUMoNg=1uDh8<;d@R_;^%i1iC=&5?Dm#H3}p0F0V*+bdp zHZfb94%zT5ES7h^eJ|X7X|lvPbD+v~-)|{gf~rK*-O($V4tPPI1iS(}RtorvQ&DED$Pc078YD=B$drgP$Sa^%e z488JLXQ5R1Pp5SGzWu485R&g-M6AHEzKJADL;v(_07ZUzwlO?(Tk+3A;(bxQ#3GfE vus6_VbKn;=pdMW@$3K5NaP~t(@6J9=cu!xGf-Mtx0@Bwt(W$t6{r-OdS4U%7 literal 7121 zcmaJ`XFwC(whcBA5d{(HVn9%;gbq^FP(qa=y(ok(AYDQiMLGs4h7x)S1R~AQks>5i zArz@mF(AE!j>rps_uc!w`|f)`lFXUO?0wFjwf5R)Pt-#lwTrZDv;Y9$B19dm4**bs zD95cd=P2J=*t?ySA1W_>wflhbevSpo;EcmP?Rx+~MZ$$&PtH=t=bx#YdI13R?I$0q zF85+60Kje!0pBxxZncsb@WxUlv#Y3Ds7(KO{nGJ!!+cs(pDJ#mnuSIP+y@EgjH znr-p{0J0f|g(@qq9W^iyN4i`i#KN=QZ`#NHqL#D8_aHiW+nK#ldJ`6+E~#6wLakC_a7)khGN~b80N`X28YE){FP3#&BVYtd6%T0iU^khn;F%_ z3U?ATq|VvGlYb@8xOg@(;gC!P2tUO5!1<_ZWafONdIDN%C_%|w`ynNIVmzOG8X0Y!na8&VLLzjmgFgzEUM?zo zo+)zon8Q6FE&IL6Gu4ME^qj{yZ{i?ZQOp+iY`zs= zu7$!?vtR*TU3Y_=&1MKSKq8as+PF%Vye9FA{K~4*^G_s@+riBlU(g~e+^$kCJh%~} zA;JyPovay?4-@4803>aBJ3~m#n5PUrQL(HZ(+N#Op0e{TtJ|Yk6p0HV$mv%-`WRkd z=We!7n=rmPb`Ic3qv*kj>0M-I6}RuyI?689eca2>+?2hB+FzV|Ly*ht)Q9r`=W8E& zZpxriql)KXff*k`?Kry7P6J-_EZj5*$iCr`b~l(Vu5m5|U9@+z-7O@e8B{W2t@M)2 zfl#2;;3Y2?LSJ^=Ap0j3$!|Li#V=TAKNgUu&OvWR5VgsLTQodtvzhvEQ$!T)uy5HRA>bt~y+ zUx3U=C=U4IlcBCM^l>o&t^$6bGHO{kn!@aicP!8kz_u($waq&V8(>8-T(lDlZlb1` zq8i?AJgjc4tUD!rtG+dqigK4QaA{?wpu)CmDB>b}NsN|8PD5sQnfBsvA2X{ax(2*5 zom{=@8D*xig0B#)RJFLkVyFyd(9?#h+8A30Nj7bdnr}x5C5_Usjg(_E-EMw061WgW zdb6Fj&DIJz3yg^b!s{4ki_M{Glc?cbRVk-nNZjYGtr#`=!gzeDU0<_X6_}*&BIi20 z;B2NF$vY-ut9_9G)7dw=iDuqSvzxq;K|bvEOZ_BiTf)kD*%bW#-Sci-n~euz9TJ-t z3`Y!JJa)yeA{KPYi=1q>@;cf%iHui|lXoY!l0>z~=4oqO`m-1X2Ijpn<3XGYK2BQo zLU|amzOnaOu3^T|i#Ali7#ij)G10H=pv;LZT^%2h2%l^PBV74M%j zT`8IWyBe!69qZ9rNLRO_HyV6@uCZ@W;^WBvW-84xgCb{5t^IJUVl2cdMx0q7Y z5*I39!hvzED7*4F!~FVS<=Dp$Ps3GSGMNhNJ%ZY^6*26782Z(%?LWQ|XE9{k%x}j# zuhGWQv!&c1BK*bF8ue@l(phW%D5vDMqecy8X1I8j6;O2f zPy81@jnM;&*#4E^|3EnGKj0OaK{?5-dgnwMF8)m#Ou^ijD~geW!bOt{F!-TdIUW|iqS#H)P; zQL}zpbmk#rjC}0{rAW{4;GEYV?_Y(HxP4Z}PjQHZ!CJT2zJ=m0i>bhICN@Io_;`A= z4^I)yEEnPk6B85F2`13;-ZUPN-))gsR{cHL%Kr1dKOziPV2%6uA-ff$v5b#UO<&Xh~$)5%cxn(G}r#V+lZ!G-c@4#1!{fTdUi z>h7gi#@mK(Q_%yul3x4unTaOX^aEim?9`W`bd}3GhVGZu#aMeIEbItd$Zt%gFdP${CjrjfE{!9@87haTct`X@6u$<9A zv~&$m=(4Yv$e&)8nO(beJFy9I5AP(+&oDGI<#+jybXE&#Pd&Q1x!1!Tqw0BDf51+w6_57&U zqRIZ;m!qW~Y&raPZ9mgL7SyXHjD0$OpHQex;Hl6@f#33J59Nt-6>K<`?Hg$NL7%+g zl&N0JRfUi~CJCgjeXxz+$6$hgHk#J6<{%i-9S6$z2~&YsE4^TaF+>{`{Dw_55ROkj zx@Pf=sxM0^3kHKZ?r)L~g2w?f%f_`tvqdL!x!l37TrZfsvjBr7kzZ9;F~Q=6fu`&$ ziFq%=H0tK{o}HoF*=w+}>gs8;qLd6#!t~NsrtZf-#+`XUDdXMAf+!0S!s>KG6$491 zc-^ya_nVaNBv{`<&*SzwYchcNe%7DGLi7g2T?_@vD#oIt<5=K{vhRB5TTq+&uBNCIh}CZTA* zhX7%92w^TWCx8ZMG1@lo3Em(=%7I%Wy^8v&?=YWe4a^;IL8Be?l4c7mMP0f|1^3Aho|yn` zqvA3kORo}O%zgPOh^4+TVNg|qGTlQ8qi%M#E&lwOZP%0H6jt$BrmU> zx?1T&+q9f+rK-c(0Cbs11K@|eRvNA6{bllT%yAr$SK$JYbRLO$>QMUw+>%*5tGKEt z71m`;#Hsf_^Hg^tz4p0ZLKvJ@pe$p-tLa`jS&Ke~Da93g|(a|L%2y88Qd(&T(f ztdjitGi1#k&vwQUcqIG%){-w;!bgA!FmNGRbF@AwnIT2vnqcmvi}9wTkb(|*+ke<` z(Ci8!_6vv13!nj^H{bG=u2o}JX1bn%zK$Aq8VdzFCi-{3FD@?TZp?NOeh%h2Eq|$~ z_FXYsTw3aZ#D?K|#bVmM9kUv%wV*|Q9f{q1*r7U|`K!JA!D-^O5-Sb!kljhm!0Y9f zW9O&kXN%>47vkFP03T$$JOSOS8rMmwv^!5u#j#0hBy>Jc^4(j1;_e@bzNJvT*4`)G z<;I>u$C{F1z~{oxS%wJk<)VbFaTmu8>fojj6oDe82%9@jR6D_VjRcY%hw>pX@6J34 zuCgc22!>jBUvc8k++6>bmBmbMA@$%EJnM_yBdyv}_Zzm~O3g?Xm^SSR> zKWjZJ5%N25abY2c)RVe&XjePF)_ycPG=-ss6e!<{@;+tIgXZd~dZjT}aq;cEf-Z0( zF)eX3k2Ob4K}HCUdXMBW&^NE<;sJGhjru?NozNx9aYFC80LL_k^Z4+7Bxs;wyg&(Pe{*HB9!UoZZWdjBCpuyY)BCDH^kZplInaS8gXo@l7o`x z<2$2;GNb1{N~cA3f+gW%LL$Z{s?ScOQZm{=6-op4%43F)T=>r#8*&E_d7l3FugXxi zSCi5dxmM=3xk#L^ga3=Ze~MIoe#-tfk|CBN_gTCSst0#Qz z@RJ3Aj*>{gJe8wHhp+kLYV&dEK&x{JsqNw*mybSlOsg(b{AVed;X8vVqWh*;^!FM@ zjmAAqWvCn555VFXinA~#gcWPI)?tUn!@5s~=mQ9=@QoB>vjpP^TFkuD<+1kt#m2IQ zy2ers{42{J!^3r*XoZ?AG$2dlC3=?t<0F=J;m*2v?*|6h=bQEf@L~At&JC8!@)xsH z+@8~cmK#1-dhc2P$U5I)Ux>q>qeO5*m%Kd`gnGXmzbaDGHP+)p*fMmkyZTjo(vxVB z4vU+svD>aB7nqP9u_Mmx*SwsmTHl6lj%Z%3U!yLj{>v(a`sufuYr{VaXa$3K@u6W?s+mEN-tJ=*RAicdfrtplN=+SR&iyPzQBF1&z8LELjQ zd0@n6M%=m~$J>i;@!VezYt8aep#mLoOry}yiF!0qyiHS;qb`|1sAmOb?#BI^@-F4! zCz_=#BE>If1=ii#P?u$p?S!?HzfSv zyL(nKXv{>pMV^fHVDsV1Qzy$LG% z%cg%5q5llplO~2IrF;!#X+7y`NdD)dSLNl}m~hEA7wS-h6IC+N+CBq57ePw_Dy8@F zi``-YgD_GLu_hXGF?lt%I7JZeyU?6mw1DbR54q@dPNAanxyfla5@uZq!}5a%&5bwC zP$aL9*>o`R=^$a5KBN+S2JS>*AIoP3ME8KOMD7w3`pXI`Rb8+APSOV+BjfH&wK0ah z?)7Nh?`A#$zofMZg3KQNP0t~f8V35-q;$^%jI*wYw@u;bs4PK6t#_LR%(~Xb>%{6` zJ{4d~D%!q(p>#dudZrNsYh^rPwc4b=uygp$wD+bb#j9g2u2M&D|Lf97Qx&NfY)Tk6 zRQtTJ+PtbBMj9Aa6jg9AVlK(2?r%S2{RV4m;_s(*FxO#j;yQIWTi@PSV;`-@UlcV#&>yZzPPuI{5h2x9U;B&{B&Wza#*A_yPg zsG2+;q~>N`IHpkJJ+WN|LzcR1-d$6h=trG8fxtDy-W#tjy-XmYEuT|;^!69bu-C-A zr|D~u6OxRR3j=-NCBzFrldZQWr`E6KiTh6<+}&Bi6*?^{;G39>v;=!3D;_Z!BC+>saz!GLt@>-sOQPJ)(3Ja&v&ECG*^W#K1`&H&o8D zorF%P5Vlcsx}=>j0tG`8p2o9BqEY{r{{OxECrfF1D91v9WCT3`a8A&@<;eFO_?&WR z7jWWdMa~7vA(cK;co5lVJ?GTl2W{!@&FA4%m+wd0ymzm(-4(V_*Uj{zU%8#SfMJ(5 z^G_VZv>>h6#a_^3EFd9k%UdZ?VYXfI_n?56Z*q(M%#V9vu&q~W3%{EWRh-FE1k|OY z_1D$p2zJL^SL|TFte4|qqyN@i#6_d7ki#(QpFhj6LrcVc`X;du#qEBic9YT`)e*7% zk~zud8xGDxq5F;eTZ`jz$dzn0WWY-)#TK}cLY>AB33u^K7#ZO(Bq&YP3YhCJ2qGSG ztsISF9ha~z+9Ccw;OO2Z;*rQU(Sy1v2RwS*=RQ>Hc5S z_Ax))G!wqV6;~Dj%fxwnxVD--?#v6VCA9{qxUo-*9WtB2K5`H>HG4{?qCafOv>VF}DlyDvg=KZM8Vm_MhbJgG+YrKY~8?2serLNDtx)jVZCu7`6G z4`h6rzo5qVQsaQePt9!nPr3;)XxCSQ^}p=pBoi8~^eY#ew)7`3ygm;h%aLi$JE$Y% z0s&=B+7`rDd^{LYU&hX7>0(PO&5hC7N{59hu2qgviDPMJeJA+*rwmJC&{*s>;A!rc zOe(_cJuzxFLRB?QE-LX;j84-VFSHCM!wS56mgSS$_KJa(0oNM~p7GjCjaMA+eLP+L zNL$pe@7=wiRBIiz^2S7~EXkPo;8}KWv|+BThND`MPC>tDFr4{Tln>FrNOBwfCbmN; z>Y~hu%1*Z+hJxS|%zP*daw=nHA(Q)*uF3Y>pVxE~k`brXRN2X9t4mHT$EW8D z7_I=w+;b^rgt3liekV|KHIt<37S=b`O01pdnvnDWaz5W}K&@r2$+DfZe?= zRiQf(c1GJCF)&rA=rhx|MveP&d+cMDYSc5w!J0PIS1|&x*BA4UlWrHZZW}nWz;7mk z@^0M*-M;R@;Y?tQD{y2mx;7po+D!AuXxr2w?E9|n=L045z>vJJ57eOBT5-QeNTG4W zFF|PE<|Dsiz+28k_oY$2mP8ZcYb6W#mjI&Xq0)|+a0TbubLyt-=MZga>*k}KDKGo0 zB~v_A`5kt;hEPQ(iW{#11?oR;a)Vp~q=JTdFsJdjn1p>kr5H5XJ=1AHz+6WMZwT5d zD`@^^O|n8PfB9ARBW$+`0~C`Km=tqjbbUy`&*%n4BaImY?r+FkSZO3X&c8VOErjY# zd~GAq%oB-#IWvrKSGJ+-E*j`u8n{cQqa-Jq+q5%sR!g55#lP3DeLCC{x9t@TG465H zdfM*=-J$7YM(CL4bYI2wdM`>-GC^}fU`OJY6v`fGgOqA8mJkdUCBg)o_7-#rwi-5< z1V;stY+^59bA5SA5vulP`ZnFE)6!R@S6*x9OV-5VDrYqUey1NrvlXSJOS?zl05msL zo=7!QcLHbmcdt)uUUq$M|6n5=bGjO#bM#uUJiGn+p7Ng+%G%K$VG_T1QMM)QhrRv# zWYpnuRZWK+6T7ZbCNe|pfs!WQS%wRimX?2|wI99%J-d!!Wg*T@_S5iIo=V%5mFkcQ z2|4cxr{T=_4y0LS$ANERmiXlU(HddSYoI{QuxZ*H6Q)VtMe3Fe-9I;Op7Z2R^a5(>tPAUemVx-p1Vc zA14k*Feh)O8n - - + + diff --git a/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb b/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb index 88830e4..5ebb707 100644 --- a/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb +++ b/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + + diff --git a/SCrawler/API/BaseObjects/InternalSettingsForm.vb b/SCrawler/API/BaseObjects/InternalSettingsForm.vb index 0f84bee..987821e 100644 --- a/SCrawler/API/BaseObjects/InternalSettingsForm.vb +++ b/SCrawler/API/BaseObjects/InternalSettingsForm.vb @@ -154,7 +154,8 @@ Namespace API.Base Dim tmpObj As Object members = GetObjectMembers(MyObject, Function(m) (m.MemberType = MemberTypes.Field Or m.MemberType = MemberTypes.Property) AndAlso - Not m.GetCustomAttribute(Of PSettingAttribute) Is Nothing) + Not m.GetCustomAttribute(Of PSettingAttribute) Is Nothing,, True, + New FComparer(Of MemberInfo)(Function(mm1, mm2) mm1.Name = mm2.Name)) providersMembersSettings = GetObjectMembers(MySettingsInstance, providersPredicate) providersMembersObj = GetObjectMembers(MyObject, providersPredicate) diff --git a/SCrawler/API/Mastodon/EditorExchangeOptions.vb b/SCrawler/API/Mastodon/EditorExchangeOptions.vb new file mode 100644 index 0000000..d0c24c9 --- /dev/null +++ b/SCrawler/API/Mastodon/EditorExchangeOptions.vb @@ -0,0 +1,22 @@ +' Copyright (C) 2023 Andy https://github.com/AAndyProgram +' This program is free software: you can redistribute it and/or modify +' it under the terms of the GNU General Public License as published by +' the Free Software Foundation, either version 3 of the License, or +' (at your option) any later version. +' +' This program is distributed in the hope that it will be useful, +' but WITHOUT ANY WARRANTY +Imports SCrawler.Plugin.Attributes +Namespace API.Mastodon + Friend Class EditorExchangeOptions : Inherits Twitter.EditorExchangeOptions + Friend Overrides Property DownloadModelMedia As Boolean + Friend Overrides Property DownloadModelProfile As Boolean + Friend Overrides Property DownloadModelSearch As Boolean + Friend Sub New(ByVal s As SiteSettings) + MyBase.New(s) + End Sub + Friend Sub New(ByVal u As UserData) + MyBase.New(u) + End Sub + End Class +End Namespace \ No newline at end of file diff --git a/SCrawler/API/Mastodon/SiteSettings.vb b/SCrawler/API/Mastodon/SiteSettings.vb index 2ff6805..1761c34 100644 --- a/SCrawler/API/Mastodon/SiteSettings.vb +++ b/SCrawler/API/Mastodon/SiteSettings.vb @@ -139,9 +139,9 @@ Namespace API.Mastodon #End Region #Region "UserOptions" Friend Overrides Sub UserOptions(ByRef Options As Object, ByVal OpenForm As Boolean) - If Options Is Nothing OrElse (Not TypeOf Options Is Twitter.EditorExchangeOptions OrElse - Not DirectCast(Options, Twitter.EditorExchangeOptions).SiteKey = MastodonSiteKey) Then _ - Options = New Twitter.EditorExchangeOptions(Me) With {.SiteKey = MastodonSiteKey} + If Options Is Nothing OrElse (Not TypeOf Options Is EditorExchangeOptions OrElse + Not DirectCast(Options, EditorExchangeOptions).SiteKey = MastodonSiteKey) Then _ + Options = New EditorExchangeOptions(Me) With {.SiteKey = MastodonSiteKey} If OpenForm Then Using f As New InternalSettingsForm(Options, Me, False) : f.ShowDialog() : End Using End If diff --git a/SCrawler/API/Twitter/EditorExchangeOptions.vb b/SCrawler/API/Twitter/EditorExchangeOptions.vb index 6ef40c0..86d5e70 100644 --- a/SCrawler/API/Twitter/EditorExchangeOptions.vb +++ b/SCrawler/API/Twitter/EditorExchangeOptions.vb @@ -7,6 +7,7 @@ ' This program is distributed in the hope that it will be useful, ' but WITHOUT ANY WARRANTY Imports SCrawler.Plugin.Attributes +Imports DModels = SCrawler.API.Twitter.UserData.DownloadModels Namespace API.Twitter Friend Class EditorExchangeOptions Private Const DefaultOffset As Integer = 100 @@ -23,6 +24,18 @@ Namespace API.Twitter ToolTip:="Existing files will be checked for duplicates and duplicates removed." & vbCr & "Works only on the first activation 'Use MD5 comparison'.", LeftOffset:=DefaultOffset)> Friend Property RemoveExistingDuplicates As Boolean = False + + Friend Overridable Property DownloadModelMedia As Boolean = False + + Friend Overridable Property DownloadModelProfile As Boolean = False + + Friend Overridable Property DownloadModelSearch As Boolean = False Private ReadOnly Property MySettings As Object Friend Sub New(ByVal s As SiteSettings) GifsDownload = s.GifsDownload.Value @@ -44,6 +57,14 @@ Namespace API.Twitter GifsPrefix = u.GifsPrefix UseMD5Comparison = u.UseMD5Comparison RemoveExistingDuplicates = u.RemoveExistingDuplicates + If Not TypeOf u Is Mastodon.UserData Then + Dim dm As DModels() = EnumExtract(Of DModels)(u.DownloadModel) + If dm.ListExists Then + DownloadModelMedia = dm.Contains(DModels.Media) + DownloadModelProfile = dm.Contains(DModels.Profile) + DownloadModelSearch = dm.Contains(DModels.Search) + End If + End If MySettings = u.HOST.Source End Sub End Class diff --git a/SCrawler/API/Twitter/UserData.vb b/SCrawler/API/Twitter/UserData.vb index 80bab06..e7588ea 100644 --- a/SCrawler/API/Twitter/UserData.vb +++ b/SCrawler/API/Twitter/UserData.vb @@ -18,11 +18,21 @@ Imports UTypes = SCrawler.API.Base.UserMedia.Types Namespace API.Twitter Friend Class UserData : Inherits UserDataBase #Region "XML names" + Private Const Name_FirstDownloadComplete As String = "FirstDownloadComplete" + Private Const Name_DownloadModel As String = "DownloadModel" Private Const Name_GifsDownload As String = "GifsDownload" Private Const Name_GifsSpecialFolder As String = "GifsSpecialFolder" Private Const Name_GifsPrefix As String = "GifsPrefix" #End Region #Region "Declarations" + Friend Enum DownloadModels As Integer + Undefined = 0 + Media = 1 + Profile = 2 + Search = 5 + End Enum + Private FirstDownloadComplete As Boolean = False + Friend Property DownloadModel As DownloadModels = DownloadModels.Undefined Friend Property GifsDownload As Boolean = True Friend Property GifsSpecialFolder As String = String.Empty Friend Property GifsPrefix As String = String.Empty @@ -53,6 +63,10 @@ Namespace API.Twitter GifsPrefix = .GifsPrefix UseMD5Comparison = .UseMD5Comparison RemoveExistingDuplicates = .RemoveExistingDuplicates + DownloadModel = DownloadModels.Undefined + If .DownloadModelMedia Then DownloadModel += DownloadModels.Media + If .DownloadModelProfile Then DownloadModel += DownloadModels.Profile + If .DownloadModelSearch Then DownloadModel += DownloadModels.Search End With End If End Sub @@ -62,25 +76,48 @@ Namespace API.Twitter _DataNames = New List(Of String) End Sub Protected Overrides Sub LoadUserInformation_OptionalFields(ByRef Container As XmlFile, ByVal Loading As Boolean) - If Loading Then - GifsDownload = Container.Value(Name_GifsDownload).FromXML(Of Boolean)(True) - GifsSpecialFolder = Container.Value(Name_GifsSpecialFolder) - If Not Container.Contains(Name_GifsPrefix) Then - GifsPrefix = "GIF_" + With Container + If Loading Then + If .Contains(Name_FirstDownloadComplete) Then + FirstDownloadComplete = .Value(Name_FirstDownloadComplete).FromXML(Of Boolean)(False) + DownloadModel = .Value(Name_DownloadModel).FromXML(Of Integer)(DownloadModels.Undefined) + Else + FirstDownloadComplete = DownloadedVideos(True) + DownloadedPictures(True) > 0 + If .Contains(Name_DownloadModel) Then + DownloadModel = .Value(Name_DownloadModel).FromXML(Of Integer)(DownloadModels.Undefined) + Else + If FirstDownloadComplete Then + If ParseUserMediaOnly Then + DownloadModel = DownloadModels.Media + Else + DownloadModel = DownloadModels.Media + DownloadModels.Profile + DownloadModels.Search + End If + Else + DownloadModel = DownloadModels.Undefined + End If + End If + End If + GifsDownload = .Value(Name_GifsDownload).FromXML(Of Boolean)(True) + GifsSpecialFolder = .Value(Name_GifsSpecialFolder) + If Not .Contains(Name_GifsPrefix) Then + GifsPrefix = "GIF_" + Else + GifsPrefix = .Value(Name_GifsPrefix) + End If + UseMD5Comparison = .Value(Name_UseMD5Comparison).FromXML(Of Boolean)(False) + RemoveExistingDuplicates = .Value(Name_RemoveExistingDuplicates).FromXML(Of Boolean)(False) + StartMD5Checked = .Value(Name_StartMD5Checked).FromXML(Of Boolean)(False) Else - GifsPrefix = Container.Value(Name_GifsPrefix) + .Add(Name_FirstDownloadComplete, FirstDownloadComplete.BoolToInteger) + .Add(Name_DownloadModel, CInt(DownloadModel)) + .Add(Name_GifsDownload, GifsDownload.BoolToInteger) + .Add(Name_GifsSpecialFolder, GifsSpecialFolder) + .Add(Name_GifsPrefix, GifsPrefix) + .Add(Name_UseMD5Comparison, UseMD5Comparison.BoolToInteger) + .Add(Name_RemoveExistingDuplicates, RemoveExistingDuplicates.BoolToInteger) + .Add(Name_StartMD5Checked, StartMD5Checked.BoolToInteger) End If - UseMD5Comparison = Container.Value(Name_UseMD5Comparison).FromXML(Of Boolean)(False) - RemoveExistingDuplicates = Container.Value(Name_RemoveExistingDuplicates).FromXML(Of Boolean)(False) - StartMD5Checked = Container.Value(Name_StartMD5Checked).FromXML(Of Boolean)(False) - Else - Container.Add(Name_GifsDownload, GifsDownload.BoolToInteger) - Container.Add(Name_GifsSpecialFolder, GifsSpecialFolder) - Container.Add(Name_GifsPrefix, GifsPrefix) - Container.Add(Name_UseMD5Comparison, UseMD5Comparison.BoolToInteger) - Container.Add(Name_RemoveExistingDuplicates, RemoveExistingDuplicates.BoolToInteger) - Container.Add(Name_StartMD5Checked, StartMD5Checked.BoolToInteger) - End If + End With End Sub #End Region #Region "Download functions" @@ -262,7 +299,20 @@ Namespace API.Twitter End If End If Next + dirs.Clear() End If + ThrowAny(Token) + If Not FirstDownloadComplete Then + _ForceSaveUserInfo = True + If DownloadModel = DownloadModels.Undefined Then + If ParseUserMediaOnly Then + DownloadModel = DownloadModels.Media + Else + DownloadModel = DownloadModels.Media + DownloadModels.Profile + DownloadModels.Search + End If + End If + End If + FirstDownloadComplete = True Catch ex As Exception ProcessException(ex, Token, $"data downloading error [{URL}]") Finally @@ -459,7 +509,8 @@ Namespace API.Twitter Private Function GetTimelineFromGalleryDL(ByVal Cache As CacheKeeper, ByVal Token As CancellationToken) As List(Of SFile) Dim command$ = String.Empty Try - Dim conf As SFile = $"{Cache.NewPath.PathWithSeparator}TwitterGdlConfig.conf" + Dim confCache As CacheKeeper = Cache.NewInstance(Of BatchFileExchanger) + Dim conf As SFile = $"{confCache.RootDirectory.PathWithSeparator}TwitterGdlConfig.conf" Dim confText$ = "{""extractor"":{""cookies"": """ & MySettings.CookiesNetscapeFile.ToString.Replace("\", "/") & """,""cookies-update"": false,""twitter"":{""cards"": false,""conversations"": true,""pinned"": false,""quoted"": false,""replies"": true,""retweets"": true,""strategy"": null,""text-tweets"": false,""twitpic"": false,""unique"": true,""users"": ""timeline"",""videos"": true}}}" If conf.Exists(SFO.Path, True, EDP.ThrowException) Then TextSaver.SaveTextToFile(confText, conf) @@ -468,8 +519,19 @@ Namespace API.Twitter Dim outList As New List(Of SFile) Dim rootDir As CacheKeeper = Cache.NewInstance Dim dir As SFile + Dim dm As List(Of DownloadModels) = EnumExtract(Of DownloadModels)(DownloadModel).ListIfNothing + Dim process As Boolean + Dim bProcess As Boolean = DownloadModel = DownloadModels.Undefined Or Not FirstDownloadComplete - Using tgdl As New TwitterGDL(Nothing, Token) With {.TempPostsList = _TempPostsList, .AutoClear = True, .AutoReset = True} + Using tgdl As New TwitterGDL(Nothing, Token) With { + .TempPostsList = _TempPostsList, + .AutoClear = True, + .AutoReset = True, + .CommandPermanent = $"chcp {BatchExecutor.UnicodeEncoding}", + .FileExchanger = confCache + } + tgdl.FileExchanger.DeleteCacheOnDispose = False + tgdl.FileExchanger.DeleteRootOnDispose = False For i As Byte = 0 To 2 dir = rootDir.NewPath dir.Exists(SFO.Path, True, EDP.ThrowException) @@ -478,20 +540,25 @@ Namespace API.Twitter command = $"""{Settings.GalleryDLFile}"" --verbose --no-download --no-skip --config ""{conf}"" --write-pages " command &= GdlGetIdFilterString() Select Case i - Case 0 : command &= $"https://twitter.com/{Name}/media" - Case 1 : command &= $"https://twitter.com/{Name}" - Case 2 : command &= $"https://twitter.com/search?q=from:{Name}+include:nativeretweets" + Case 0 : command &= $"https://twitter.com/{Name}/media" : process = bProcess Or dm.Contains(DownloadModels.Media) + Case 1 : command &= $"https://twitter.com/{Name}" : process = bProcess Or dm.Contains(DownloadModels.Profile) + Case 2 : command &= $"https://twitter.com/search?q=from:{Name}+include:nativeretweets" : process = bProcess Or dm.Contains(DownloadModels.Search) + Case Else : process = False End Select '#If DEBUG Then 'Debug.WriteLine(command) '#End If - tgdl.Execute(command) + ThrowAny(Token) + If process Then tgdl.Execute(command) + ThrowAny(Token) Next End Using + dm.Clear() Return outList Catch ex As Exception - Return ErrorsDescriber.Execute(EDP.SendToLog, ex, $"{ToStringForLog()}: GetTimelineFromGalleryDL({command})") + ProcessException(ex, Token, $"{ToStringForLog()}: GetTimelineFromGalleryDL({command})") + Return Nothing End Try End Function Private Function GdlGetIdFilterString() As String diff --git a/SCrawler/Download/DownloadProgress.vb b/SCrawler/Download/DownloadProgress.vb index 9386d6d..c768980 100644 --- a/SCrawler/Download/DownloadProgress.vb +++ b/SCrawler/Download/DownloadProgress.vb @@ -24,6 +24,7 @@ Namespace DownloadObjects Private WithEvents BTT_STOP As Button Private WithEvents BTT_OPEN As Button Private ReadOnly PR_MAIN As ProgressBar + Private ReadOnly PR_PRE As ProgressBar Private ReadOnly LBL_INFO As Label Private ReadOnly Icon As PictureBox #End Region @@ -39,6 +40,7 @@ Namespace DownloadObjects TP_MAIN.ColumnCount = 1 TP_CONTROLS = New TableLayoutPanel With {.Margin = New Padding(0), .Dock = DockStyle.Fill} PR_MAIN = New ProgressBar With {.Dock = DockStyle.Fill} + PR_PRE = New ProgressBar With {.Dock = DockStyle.Fill} LBL_INFO = New Label With {.Text = String.Empty, .Dock = DockStyle.Fill} Icon = New PictureBox With { .SizeMode = PictureBoxSizeMode.Zoom, @@ -66,7 +68,8 @@ Namespace DownloadObjects With TP_CONTROLS .ColumnStyles.Add(New ColumnStyle(SizeType.Absolute, 30)) .ColumnStyles.Add(New ColumnStyle(SizeType.Absolute, 30)) - .ColumnStyles.Add(New ColumnStyle(SizeType.Absolute, 150)) + .ColumnStyles.Add(New ColumnStyle(SizeType.Absolute, 75)) + .ColumnStyles.Add(New ColumnStyle(SizeType.Absolute, 75)) '150 .ColumnStyles.Add(New ColumnStyle(SizeType.Percent, 100)) .ColumnCount = .ColumnStyles.Count .RowStyles.Add(New RowStyle(SizeType.Percent, 100)) @@ -74,8 +77,9 @@ Namespace DownloadObjects With .Controls If Not img Is Nothing Then .Add(Icon, 0, 0) .Add(BTT_STOP, 1, 0) - .Add(PR_MAIN, 2, 0) - .Add(LBL_INFO, 3, 0) + .Add(PR_PRE, 2, 0) + .Add(PR_MAIN, 3, 0) + .Add(LBL_INFO, 4, 0) End With End With TP_MAIN.Controls.Add(TP_CONTROLS, 0, 0) @@ -90,7 +94,8 @@ Namespace DownloadObjects .Add(New ColumnStyle(SizeType.Absolute, 30)) .Add(New ColumnStyle(SizeType.Absolute, 30)) .Add(New ColumnStyle(SizeType.Absolute, 30)) - .Add(New ColumnStyle(SizeType.Percent, 100)) + .Add(New ColumnStyle(SizeType.Percent, 50)) + .Add(New ColumnStyle(SizeType.Percent, 50)) '100 End With .ColumnCount = .ColumnStyles.Count .RowStyles.Add(New RowStyle(SizeType.Percent, 50)) @@ -100,7 +105,8 @@ Namespace DownloadObjects .Add(BTT_START, 1, 0) .Add(BTT_STOP, 2, 0) .Add(BTT_OPEN, 3, 0) - .Add(PR_MAIN, 4, 0) + .Add(PR_PRE, 4, 0) + .Add(PR_MAIN, 5, 0) End With End With With TP_MAIN @@ -115,7 +121,7 @@ Namespace DownloadObjects End If With Job - .Progress = New MyProgressExt(PR_MAIN, LBL_INFO) With {.ResetProgressOnMaximumChanges = False} + .Progress = New MyProgressExt(PR_MAIN, PR_PRE, LBL_INFO) With {.ResetProgressOnMaximumChanges = False} With DirectCast(.Progress, MyProgressExt) AddHandler .ProgressChanged, AddressOf JobProgress_ProgressChanged AddHandler .MaximumChanged, AddressOf JobProgress_MaximumChanged @@ -197,7 +203,7 @@ Namespace DownloadObjects End Sub Private Sub JobProgress_Progress0Changed(ByVal Sender As Object, ByVal e As ProgressEventArgs) If Not Job.Type = Download.SavedPosts Then - MainProgress.Value0 = DirectCast(Sender, MyProgressExt).Value0 + MainProgress.Value0 = DirectCast(Job.Progress, MyProgressExt).Value0 MainProgress.Perform0(0) End If End Sub diff --git a/SCrawler/MainFrame.Designer.vb b/SCrawler/MainFrame.Designer.vb index 627c115..a36f9d3 100644 --- a/SCrawler/MainFrame.Designer.vb +++ b/SCrawler/MainFrame.Designer.vb @@ -92,6 +92,7 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form Me.BTT_DONATE = New System.Windows.Forms.ToolStripButton() Me.Toolbar_BOTTOM = New System.Windows.Forms.StatusStrip() Me.BTT_PR_INFO = New System.Windows.Forms.ToolStripStatusLabel() + Me.PR_PRE = New System.Windows.Forms.ToolStripProgressBar() Me.PR_MAIN = New System.Windows.Forms.ToolStripProgressBar() Me.LBL_JOBS_COUNT = New System.Windows.Forms.ToolStripStatusLabel() Me.LBL_STATUS = New System.Windows.Forms.ToolStripStatusLabel() @@ -122,10 +123,10 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form Me.BTT_TRAY_SILENT_MODE = New System.Windows.Forms.ToolStripMenuItem() Me.BTT_TRAY_FEED_SHOW = New System.Windows.Forms.ToolStripMenuItem() Me.BTT_TRAY_CHANNELS = New System.Windows.Forms.ToolStripMenuItem() + Me.BTT_TRAY_DOWNLOADER = New System.Windows.Forms.ToolStripMenuItem() Me.BTT_TRAY_SHOW_HIDE = New System.Windows.Forms.ToolStripMenuItem() Me.BTT_TRAY_CLOSE = New System.Windows.Forms.ToolStripMenuItem() Me.BTT_TRAY_CLOSE_NO_SCRIPT = New System.Windows.Forms.ToolStripMenuItem() - Me.BTT_TRAY_DOWNLOADER = New System.Windows.Forms.ToolStripMenuItem() SEP_1 = New System.Windows.Forms.ToolStripSeparator() SEP_2 = New System.Windows.Forms.ToolStripSeparator() CONTEXT_SEP_1 = New System.Windows.Forms.ToolStripSeparator() @@ -633,7 +634,7 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form ' 'Toolbar_BOTTOM ' - Me.Toolbar_BOTTOM.Items.AddRange(New System.Windows.Forms.ToolStripItem() {Me.BTT_PR_INFO, Me.PR_MAIN, Me.LBL_JOBS_COUNT, Me.LBL_STATUS}) + Me.Toolbar_BOTTOM.Items.AddRange(New System.Windows.Forms.ToolStripItem() {Me.BTT_PR_INFO, Me.PR_PRE, Me.PR_MAIN, Me.LBL_JOBS_COUNT, Me.LBL_STATUS}) Me.Toolbar_BOTTOM.Location = New System.Drawing.Point(0, 439) Me.Toolbar_BOTTOM.Name = "Toolbar_BOTTOM" Me.Toolbar_BOTTOM.Size = New System.Drawing.Size(934, 22) @@ -647,6 +648,12 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form Me.BTT_PR_INFO.Padding = New System.Windows.Forms.Padding(0, 0, 3, 0) Me.BTT_PR_INFO.Size = New System.Drawing.Size(19, 17) ' + 'PR_PRE + ' + Me.PR_PRE.Name = "PR_PRE" + Me.PR_PRE.Size = New System.Drawing.Size(100, 16) + Me.PR_PRE.Visible = False + ' 'PR_MAIN ' Me.PR_MAIN.Name = "PR_MAIN" @@ -865,6 +872,13 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form Me.BTT_TRAY_CHANNELS.Size = New System.Drawing.Size(170, 22) Me.BTT_TRAY_CHANNELS.Text = "Channels" ' + 'BTT_TRAY_DOWNLOADER + ' + Me.BTT_TRAY_DOWNLOADER.Image = Global.SCrawler.My.Resources.Resources.ArrowDownPic_Blue_24 + Me.BTT_TRAY_DOWNLOADER.Name = "BTT_TRAY_DOWNLOADER" + Me.BTT_TRAY_DOWNLOADER.Size = New System.Drawing.Size(170, 22) + Me.BTT_TRAY_DOWNLOADER.Text = "Downloader" + ' 'BTT_TRAY_SHOW_HIDE ' Me.BTT_TRAY_SHOW_HIDE.Image = Global.SCrawler.My.Resources.Resources.ApplicationPic_16 @@ -893,13 +907,6 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form Me.BTT_TRAY_CLOSE_NO_SCRIPT.ToolTipText = "Close the program without executing the script" Me.BTT_TRAY_CLOSE_NO_SCRIPT.Visible = False ' - 'BTT_TRAY_DOWNLOADER - ' - Me.BTT_TRAY_DOWNLOADER.Image = Global.SCrawler.My.Resources.Resources.ArrowDownPic_Blue_24 - Me.BTT_TRAY_DOWNLOADER.Name = "BTT_TRAY_DOWNLOADER" - Me.BTT_TRAY_DOWNLOADER.Size = New System.Drawing.Size(170, 22) - Me.BTT_TRAY_DOWNLOADER.Text = "Downloader" - ' 'MainFrame ' Me.AutoScaleDimensions = New System.Drawing.SizeF(6.0!, 13.0!) @@ -1005,4 +1012,5 @@ Partial Public Class MainFrame : Inherits System.Windows.Forms.Form Friend WithEvents MENU_DOWN_ALL As ToolStripDropDownButton Private WithEvents BTT_TRAY_CHANNELS As ToolStripMenuItem Private WithEvents BTT_TRAY_DOWNLOADER As ToolStripMenuItem + Private WithEvents PR_PRE As ToolStripProgressBar End Class \ No newline at end of file diff --git a/SCrawler/MainFrame.vb b/SCrawler/MainFrame.vb index e6b4ec4..74cf8e0 100644 --- a/SCrawler/MainFrame.vb +++ b/SCrawler/MainFrame.vb @@ -58,7 +58,7 @@ Public Class MainFrame YouTube.MyCache = Settings.Cache YouTube.MyYouTubeSettings = New YouTube.YTSettings_Internal UpdateYouTubeSettings() - MainProgress = New MyProgressExt(Toolbar_BOTTOM, PR_MAIN, LBL_STATUS, "Downloading profiles' data") With { + MainProgress = New MyProgressExt(Toolbar_BOTTOM, PR_MAIN, PR_PRE, LBL_STATUS, "Downloading profiles' data") With { .ResetProgressOnMaximumChanges = False, .Visible = False} Downloader = New TDownloader InfoForm = New DownloadedInfoForm diff --git a/SCrawler/My Project/AssemblyInfo.vb b/SCrawler/My Project/AssemblyInfo.vb index 6e6b29b..e6f9587 100644 --- a/SCrawler/My Project/AssemblyInfo.vb +++ b/SCrawler/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + + diff --git a/SCrawler/MyProgressExt.vb b/SCrawler/MyProgressExt.vb index d51795b..59602e1 100644 --- a/SCrawler/MyProgressExt.vb +++ b/SCrawler/MyProgressExt.vb @@ -114,62 +114,71 @@ Friend Class MyProgressExt : Inherits MyProgress End If End RaiseEvent End Event + Private WithEvents PR_PRE As MyProgress + Private Sub PR_PRE_ProgressChanged(ByVal Sender As Object, ByVal e As ProgressEventArgs) Handles PR_PRE.ProgressChanged + RaiseEvent Progress0Changed(Sender, e) + End Sub + Private Sub PR_PRE_MaximumChanged(ByVal Sender As Object, ByVal e As ProgressEventArgs) Handles PR_PRE.MaximumChanged + RaiseEvent Maximum0Changed(Sender, e) + End Sub Friend Sub New() _Progress0ChangedEventHandlers = New List(Of EventHandler(Of ProgressEventArgs)) _Maximum0ChangedEventHandlers = New List(Of EventHandler(Of ProgressEventArgs)) End Sub - Friend Sub New(ByRef StatusStrip As StatusStrip, ByRef ProgressBar As ToolStripProgressBar, ByRef Label As ToolStripStatusLabel, + Friend Sub New(ByRef StatusStrip As StatusStrip, ByRef ProgressBar As ToolStripProgressBar, ByRef ProgressBarPre As ToolStripProgressBar, ByRef Label As ToolStripStatusLabel, Optional ByVal Information As String = Nothing) MyBase.New(StatusStrip, ProgressBar, Label, Information) + PR_PRE = New MyProgress(StatusStrip, ProgressBarPre, Nothing) With {.PerformMod = 10, .ResetProgressOnMaximumChanges = False} _Progress0ChangedEventHandlers = New List(Of EventHandler(Of ProgressEventArgs)) _Maximum0ChangedEventHandlers = New List(Of EventHandler(Of ProgressEventArgs)) End Sub - Friend Sub New(ByRef ProgressBar As ProgressBar, ByRef Label As Label, Optional ByVal Information As String = Nothing) + Friend Sub New(ByRef ProgressBar As ProgressBar, ByRef ProgressBarPre As ProgressBar, ByRef Label As Label, Optional ByVal Information As String = Nothing) MyBase.New(ProgressBar, Label, Information) + PR_PRE = New MyProgress(ProgressBarPre, Nothing) With {.PerformMod = 10, .ResetProgressOnMaximumChanges = False} _Progress0ChangedEventHandlers = New List(Of EventHandler(Of ProgressEventArgs)) _Maximum0ChangedEventHandlers = New List(Of EventHandler(Of ProgressEventArgs)) End Sub - Private _Maximum0 As Double = 0 Friend Property Maximum0 As Double Get - Return _Maximum0 + Return PR_PRE.Maximum End Get Set(ByVal v As Double) - Dim b As Boolean = Not _Maximum0 = v - _Maximum0 = v - If ResetProgressOnMaximumChanges Then Value0 = 0 - If b Then RaiseEvent Maximum0Changed(Me, Nothing) + PR_PRE.Maximum = v + End Set + End Property + Friend Property Value0 As Double + Get + Return PR_PRE.Value + End Get + Set(ByVal v As Double) + PR_PRE.Value = v End Set End Property - Friend Property Value0 As Double = 0 Friend Sub Perform0(Optional ByVal Value As Double = 1) - Value0 += Value - If Perform(0, 10, False, False) Then RaiseEvent Progress0Changed(Me, Nothing) + PR_PRE.Perform(Value) End Sub - Public Overloads Overrides Sub Perform(Optional ByVal Value As Double = 1) - If Perform(Value, PerformMod, True, True) Then OnProgressChanged() - End Sub - Public Overloads Function Perform(ByVal Value As Double, ByVal pm As Integer, ByVal SetText As Boolean, ByVal InvokeProgressChangeHandler As Boolean) As Boolean - Me.Value += Value - If Me.Value < 0 Then Me.Value = 0 - Dim v# = Me.Value + Value0 - Dim m# = Maximum + Maximum0 - If pm = 0 OrElse (v Mod pm) = 0 OrElse v = m Then PerformImpl(GetPercentage(v, m), SetText, InvokeProgressChangeHandler) : Return True - Return False - End Function Public Overrides Sub Done() - Value0 = Maximum0 + PR_PRE.Done() MyBase.Done() End Sub Public Overrides Sub Reset() MyBase.Reset() - Value0 = 0 - Maximum0 = 0 + PR_PRE.Done() End Sub + Public Overrides Property Visible(Optional ByVal ProgressBar As Boolean = True, Optional ByVal Label As Boolean = True) As Boolean + Get + Return MyBase.Visible(ProgressBar, Label) + End Get + Set(ByVal _Visible As Boolean) + MyBase.Visible(ProgressBar, Label) = _Visible + PR_PRE.Visible(ProgressBar, Label) = _Visible + End Set + End Property Protected Overrides Sub Dispose(ByVal disposing As Boolean) If Not disposedValue And disposing Then _Progress0ChangedEventHandlers.Clear() _Maximum0ChangedEventHandlers.Clear() + PR_PRE.Dispose() End If MyBase.Dispose(disposing) End Sub diff --git a/SCrawler/PluginsEnvironment/Attributes/Attributes.vb b/SCrawler/PluginsEnvironment/Attributes/Attributes.vb index 964069d..a046402 100644 --- a/SCrawler/PluginsEnvironment/Attributes/Attributes.vb +++ b/SCrawler/PluginsEnvironment/Attributes/Attributes.vb @@ -8,7 +8,7 @@ ' but WITHOUT ANY WARRANTY Imports System.Runtime.CompilerServices Namespace Plugin.Attributes - Public Enum SettingAddress : Both : Settings : User : End Enum + Public Enum SettingAddress : Both : Settings : User : None : End Enum Public Class PSettingAttribute : Inherits Attribute Public Number As Integer = 0 Friend ReadOnly Name As String diff --git a/SCrawler/SCrawler.vbproj b/SCrawler/SCrawler.vbproj index 5a6cbd3..39dd0d8 100644 --- a/SCrawler/SCrawler.vbproj +++ b/SCrawler/SCrawler.vbproj @@ -178,6 +178,7 @@ + SettingsForm.vb