diff --git a/Changelog.md b/Changelog.md index 8215e13..ffe50d7 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,12 +1,26 @@ +# 2025.8.1.0 + +*2025-08-01* + +- Added + - Sites: + - Reddit: **bypass error `429`** + - Twitter: **[large profile option](https://github.com/AAndyProgram/SCrawler/wiki/Settings#twitter-user-settings) in user settings** + - Minor improvements +- Updated + - yt-dlp up to version **2025.27.21** + - gallery-dl up to version **1.30.2** +- Fixed + - Reddit: in some cases crossposts don't download + - Minor bugs + # 2025.7.18.0 *2025-07-18* - Added - Sites: - - OnlyFans: - - **bypass unpurchased videos** - - support for GIF files + - OnlyFans: support for GIF files - Reddit: extended `429` error handling - Xhamster: support for downloading 'moments' - Minor improvements @@ -14,6 +28,7 @@ - yt-dlp up to version **2025.06.30** - gallery-dl up to version **1.30.0** - Fixed + - OnlyFans: **hanging on purchased content** - Minor bugs # 2025.6.12.0 diff --git a/SCrawler.YouTube/My Project/AssemblyInfo.vb b/SCrawler.YouTube/My Project/AssemblyInfo.vb index d4175e3..c0e9251 100644 --- a/SCrawler.YouTube/My Project/AssemblyInfo.vb +++ b/SCrawler.YouTube/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + + diff --git a/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb b/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb index e5e6b0c..ce12c00 100644 --- a/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb +++ b/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb @@ -823,7 +823,9 @@ Namespace API.YouTube.Objects 'cmd = $"yt-dlp -f ""{cmd}""" 'cmd = $"yt-dlp -f {cmd}" cmd = $"{YTDLP_NAME} -f {cmd}" - If Not MyYouTubeSettings.ReplaceModificationDate Then cmd &= " --no-mtime" + 'yt-dlp 2025.07.21 + 'If Not MyYouTubeSettings.ReplaceModificationDate Then cmd &= " --no-mtime" + cmd &= $" --{IIf(MyYouTubeSettings.ReplaceModificationDate.Value, String.Empty, "no-")}mtime" cmd.StringAppend(formats, " ") cmd.StringAppend(subs, " ") cmd.StringAppend(YouTubeFunctions.GetCookiesCommand(WithCookies, YouTubeCookieNetscapeFile), " ") diff --git a/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb b/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb index a4985af..3bcb338 100644 --- a/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb +++ b/SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + + diff --git a/SCrawler/API/Instagram/UserData.vb b/SCrawler/API/Instagram/UserData.vb index 2131beb..b092861 100644 --- a/SCrawler/API/Instagram/UserData.vb +++ b/SCrawler/API/Instagram/UserData.vb @@ -412,6 +412,7 @@ Namespace API.Instagram ThrowAny(Token) HasError = False Dim dt As Func(Of Boolean) = Function() (CBool(MySiteSettings.DownloadTimeline.Value) And GetTimeline) Or IsSavedPosts + If FirstLoadingDone Then LastCursor = String.Empty If dt.Invoke And Not LastCursor.IsEmptyString Then s = IIf(IsSavedPosts, Sections.SavedPosts, Sections.Timeline) upClaimRequest.Invoke diff --git a/SCrawler/API/Reddit/SiteSettings.vb b/SCrawler/API/Reddit/SiteSettings.vb index ea3e8fb..905da5e 100644 --- a/SCrawler/API/Reddit/SiteSettings.vb +++ b/SCrawler/API/Reddit/SiteSettings.vb @@ -15,21 +15,32 @@ Imports PersonalUtilities.Tools.Web.Clients.Base Imports PersonalUtilities.Tools.Web.Documents.JSON Imports PersonalUtilities.Functions.XML Imports PersonalUtilities.Functions.RegularExpressions +Imports DN = SCrawler.API.Base.DeclaredNames Imports DownDetector = SCrawler.API.Base.DownDetector Imports Download = SCrawler.Plugin.ISiteSettings.Download Namespace API.Reddit - + Friend Class SiteSettings : Inherits SiteSettingsBase : Implements DownDetector.IDownDetector #Region "Declarations" #Region "Authorization" + Private Const ApiClientID_Default As String = "dYctRA-SIJxyykHe27lGZg" + Private Const ApiClientSecret_Default As String = "_5D6KzplRPDga-es1YlpzDIe9hiFlg" Friend ReadOnly Property AuthUserName As PropertyValue Friend ReadOnly Property AuthPassword As PropertyValue Friend ReadOnly Property ApiClientID As PropertyValue + Private Function ApiClientID_SetDefault() As Boolean + ApiClientID.Value = ApiClientID_Default + Return True + End Function Friend ReadOnly Property ApiClientSecret As PropertyValue + Private Function ApiClientSecret_SetDefault() As Boolean + ApiClientSecret.Value = ApiClientSecret_Default + Return True + End Function + Friend ReadOnly Property UseM3U8 As PropertyValue - + Friend ReadOnly Property CheckImage As PropertyValue - + Friend ReadOnly Property CheckImageReturnOrig As PropertyValue + + Friend ReadOnly Property ConcurrentDownloads As PropertyValue #End Region #Region "IDownDetector Support" Private ReadOnly Property IDownDetector_Value As Integer Implements DownDetector.IDownDetector.Value @@ -160,6 +174,7 @@ Namespace API.Reddit UseM3U8 = New PropertyValue(True) CheckImage = New PropertyValue(False) CheckImageReturnOrig = New PropertyValue(True) + ConcurrentDownloads = New PropertyValue(1) MDD = New MyDownDetector(Me) @@ -167,10 +182,13 @@ Namespace API.Reddit ImageVideoContains = "reddit.com" UserRegex = RParams.DM("[htps:/]{7,8}.*?reddit.com/([user]{1,4})/([^/\?&]+)", 0, RegexReturn.ListByMatch, EDP.ReturnValue) End Sub - Private Const SettingsVersionCurrent As Integer = 2 + Private Const SettingsVersionCurrent As Integer = 3 Friend Overrides Sub EndInit() If CInt(SettingsVersion.Value) < SettingsVersionCurrent Then SettingsVersion.Value = SettingsVersionCurrent + UseM3U8.Value = True + CheckImage.Value = False + CheckImageReturnOrig.Value = True BearerTokenUseCurl.Value = False End If MyBase.EndInit() @@ -208,6 +226,7 @@ Namespace API.Reddit End Sub End Class Friend Property SessionInterrupted As Boolean = False + Friend Property RequestCount As Integer = 0 Friend Overrides Function ReadyToDownload(ByVal What As Download) As Boolean If What = Download.Main Then Return Not SessionInterrupted @@ -223,6 +242,7 @@ Namespace API.Reddit End Function Friend Overrides Sub DownloadDone(ByVal What As Download) SessionInterrupted = False + RequestCount = 0 MDD.Reset() MyBase.DownloadDone(What) End Sub diff --git a/SCrawler/API/Reddit/UserData.vb b/SCrawler/API/Reddit/UserData.vb index b8113c8..9d49ef6 100644 --- a/SCrawler/API/Reddit/UserData.vb +++ b/SCrawler/API/Reddit/UserData.vb @@ -8,19 +8,20 @@ ' but WITHOUT ANY WARRANTY Imports System.Net Imports System.Threading +Imports PersonalUtilities.Functions.RegularExpressions +Imports PersonalUtilities.Functions.XML +Imports PersonalUtilities.Tools.ImageRenderer +Imports PersonalUtilities.Tools.Web.Clients +Imports PersonalUtilities.Tools.Web.Clients.Base +Imports PersonalUtilities.Tools.Web.Documents.JSON Imports SCrawler.API.Base Imports SCrawler.API.Reddit.RedditViewExchange Imports SCrawler.API.YouTube.Objects Imports SCrawler.Plugin.Hosts -Imports PersonalUtilities.Functions.XML -Imports PersonalUtilities.Functions.RegularExpressions -Imports PersonalUtilities.Tools.ImageRenderer -Imports PersonalUtilities.Tools.Web.Clients -Imports PersonalUtilities.Tools.Web.Documents.JSON +Imports CPeriod = SCrawler.API.Reddit.IRedditView.Period +Imports CView = SCrawler.API.Reddit.IRedditView.View Imports UStates = SCrawler.API.Base.UserMedia.States Imports UTypes = SCrawler.API.Base.UserMedia.Types -Imports CView = SCrawler.API.Reddit.IRedditView.View -Imports CPeriod = SCrawler.API.Reddit.IRedditView.Period Namespace API.Reddit Friend Class UserData : Inherits UserDataBase : Implements IChannelLimits, IRedditView #Region "Declarations" @@ -269,6 +270,8 @@ Namespace API.Reddit End If End With + Responser.ProcessExceptionDecision = AddressOf Err429Process + _TotalPostsDownloaded = 0 If IsSavedPosts Then Responser.DecodersError = EDP.ReturnValue @@ -304,6 +307,7 @@ Namespace API.Reddit #End Region #Region "Download Functions (User, Channel)" Private Err429Count As Integer = 0 + Private Err429TryAgain As Boolean = False Private _TotalPostsDownloaded As Integer = 0 Private ReadOnly _CrossPosts As List(Of String) Private Const SiteGfycatKey As String = "gfycat" @@ -311,6 +315,28 @@ Namespace API.Reddit Private Const Node_CrosspostRootId As String = "crosspostRootId" Private Const Node_CrosspostParentId As String = "crosspostParentId" Private Const Node_CrosspostParent As String = "crosspost_parent" + Private Sub Wait429() + With MySiteSettings + If Not Err429TryAgain Then .RequestCount += 1 + Err429TryAgain = False + If (.RequestCount Mod 100) = 0 Then Thread.Sleep(60100) + End With + End Sub + Private Function Err429Process(ByVal Status As IResponserStatus, ByVal NullArg As Object, ByVal CurrErr As ErrorsDescriber) As ErrorsDescriber + If Not Status Is Nothing AndAlso Status.StatusCode = 429 Then + If Err429Count = 0 Then + Err429Count += 1 + MySiteSettings.RequestCount = 100 + Err429TryAgain = True + Return EDP.ReturnValue + End If + End If + Return CurrErr + End Function + Private Sub Err429Reset() + Err429Count = 0 + Err429TryAgain = False + End Sub Private Sub DownloadDataUser(ByVal POST As String, ByVal Token As CancellationToken) Dim eObj% = 0 Dim round% = 0 @@ -331,8 +357,10 @@ Namespace API.Reddit 'URL = $"https://gateway.reddit.com/desktopapi/v1/user/{NameTrue}/posts?rtj=only&allow_quarantined=true&allow_over18=1&include=identity&after={POST}&dist=25&sort={View}&t={Period}&layout=classic" URL = $"https://oauth.reddit.com/user/{NameTrue}/submitted.json?rtj=only&allow_quarantined=true&allow_over18=1&include=identity&after={POST}&dist=25&sort={View}&t={Period}&layout=classic" ThrowAny(Token) + Wait429() Dim r$ = Responser.GetResponse(URL) If Not r.IsEmptyString Then + Err429Reset() Using w As EContainer = JsonDocument.Parse(r).XmlIfNothing If w.Count > 0 Then 'n = w.GetNode(JsonNodesJson) @@ -347,6 +375,7 @@ Namespace API.Reddit If CheckNode(.Self) Then 'Obtain post ID + PostID = String.Empty PostTmp = .Value("name") '.Name If PostTmp.IsEmptyString Then PostTmp = .Value("id") If PostTmp.IsEmptyString Then Continue For @@ -354,8 +383,9 @@ Namespace API.Reddit If IsCrossPost(.Self) Then _CrossPosts.ListAddList({ .Value(Node_CrosspostRootId), .Value(Node_CrosspostParentId), - .Value(Node_CrosspostParent)}, LNC) - Continue For + .Value(Node_CrosspostParent), + PostTmp}, LNC) + If ParseUserMediaOnly Then Continue For Else If Not _CrossPosts.Contains(PostTmp) Then PostID = PostTmp : PostTmp = String.Empty End If @@ -384,6 +414,8 @@ Namespace API.Reddit End Using If POST.IsEmptyString And ExistsDetected Then Exit Sub If Not _PostID().IsEmptyString And NewPostDetected Then DownloadDataUser(_PostID(), Token) + ElseIf Err429TryAgain Then + Continue Do End If _completed = True Catch ex As Exception @@ -420,9 +452,11 @@ Namespace API.Reddit End If ThrowAny(Token) + Wait429() Dim r$ = Responser.GetResponse(URL) - If IsSavedPosts Then Err429Count = 0 + 'If IsSavedPosts Then Err429Count = 0 If Not r.IsEmptyString Then + Err429Reset() Using w As EContainer = JsonDocument.Parse(r).XmlIfNothing If w.Count > 0 Then n = w.GetNode(ChannelJsonNodes) @@ -479,6 +513,8 @@ Namespace API.Reddit End Using If POST.IsEmptyString And ExistsDetected Then Exit Sub If Not PostID.IsEmptyString And NewPostDetected Then DownloadDataChannel(PostID, Token) + ElseIf Err429TryAgain Then + Continue Do End If _completed = True Catch ex As Exception @@ -496,11 +532,13 @@ Namespace API.Reddit End Sub #End Region #Region "GetUserInfo" - Private Sub GetUserInfo() + Private Sub GetUserInfo(Optional ByVal Round As Integer = 0) Try If Not IsSavedPosts And ChannelInfo Is Nothing Then + Wait429() Dim r$ = Responser.GetResponse($"https://reddit.com/{IIf(IsChannel, "r", "user")}/{NameTrue}/about.json",, EDP.ReturnValue) If Not r.IsEmptyString Then + Err429Reset() Using j As EContainer = JsonDocument.Parse(r) If Not j Is Nothing AndAlso j.Contains({"data", "subreddit"}) Then If ID.IsEmptyString Then ID = j.Value({"data"}, "id") @@ -516,6 +554,8 @@ Namespace API.Reddit End With End If End Using + ElseIf Err429TryAgain And Round < 2 Then + GetUserInfo(Round + 1) End If End If Catch ex As Exception @@ -631,16 +671,21 @@ Namespace API.Reddit Else Dim tPostId$ = e.Value(Node_CrosspostParent).IfNullOrEmpty(e.Value(Node_CrosspostParentId)).IfNullOrEmpty(e.Value(Node_CrosspostRootId)) If Not PostID.IsEmptyString Then - Dim r$ = Responser.GetResponse($"https://www.reddit.com/comments/{tPostId.Split("_").LastOrDefault}/.json",, EDP.ReturnValue) - If Not r.IsEmptyString Then - Using j As EContainer = JsonDocument.Parse(r, EDP.ReturnValue) - If j.ListExists Then - With j.ItemF({0, "data", "children", 0, "data"}) - If .ListExists Then added = ParseContainer(.Self, PostID, PostDate, UserID, False, PostText) - End With - End If - End Using - End If + For ri% = 0 To 1 + Wait429() + Dim r$ = Responser.GetResponse($"https://www.reddit.com/comments/{tPostId.Split("_").LastOrDefault}/.json",, EDP.ReturnValue) + If Not r.IsEmptyString Then + Err429Reset() + Using j As EContainer = JsonDocument.Parse(r, EDP.ReturnValue) + If j.ListExists Then + With j.ItemF({0, "data", "children", 0, "data"}) + If .ListExists Then added = ParseContainer(.Self, PostID, PostDate, UserID, False, PostText) + End With + End If + End Using + Exit For + End If + Next End If End If End If @@ -906,7 +951,10 @@ Namespace API.Reddit End If Continue For Else + Wait429() r = Responser.GetResponse(m.URL,, e) + If r.IsEmptyString And Err429TryAgain Then _repeatForRedgifs = True + If Not r.IsEmptyString Then Err429Reset() End If Loop While _repeatForRedgifs Else @@ -944,11 +992,13 @@ Namespace API.Reddit RedGifsResponser = RedGifsHost.Responser.Copy Dim respNoHeaders As Responser = Responser.Copy Dim m As UserMedia, m2 As UserMedia - Dim r$, url$ + Dim r$ = String.Empty, url$ + Dim ri As Byte Dim j As EContainer Dim lastCount%, li% Dim rv As New ErrorsDescriber(EDP.ReturnValue) respNoHeaders.Headers.Clear() + respNoHeaders.ProcessExceptionDecision = AddressOf Err429Process ProgressPre.ChangeMax(_ContentList.Count) For i% = 0 To _ContentList.Count - 1 m = _ContentList(i) @@ -956,9 +1006,14 @@ Namespace API.Reddit If m.State = UStates.Missing AndAlso Not m.Post.ID.IsEmptyString Then ThrowAny(Token) url = $"https://www.reddit.com/comments/{m.Post.ID.Split("_").LastOrDefault}/.json" - r = Responser.GetResponse(url,, rv) - If r.IsEmptyString Then r = respNoHeaders.GetResponse(url,, rv) + For ri = 0 To 1 + Wait429() + r = Responser.GetResponse(url,, rv) + If r.IsEmptyString Then Wait429() : r = respNoHeaders.GetResponse(url,, rv) + If Not (r.IsEmptyString And Err429TryAgain) Then Exit For + Next If Not r.IsEmptyString Then + Err429Reset() j = JsonDocument.Parse(r, rv) If Not j Is Nothing Then If j.Count > 0 Then @@ -1101,13 +1156,22 @@ Namespace API.Reddit ElseIf .StatusCode = HttpStatusCode.InternalServerError Then '500 If Not IsNothing(EObj) AndAlso IsNumeric(EObj) AndAlso CInt(EObj) = HttpStatusCode.InternalServerError Then Return 1 Return HttpStatusCode.InternalServerError - ElseIf .StatusCode = 429 And IsSavedPosts And Err429Count = 0 Then '429 (saved) - Err429Count += 1 - Return 429 + 'ElseIf .StatusCode = 429 And IsSavedPosts And Err429Count = 0 Then '429 (saved) + ' Err429Count += 1 + ' Return 429 ElseIf .StatusCode = 429 Then '429 (all) + 'If ((Not IsSavedPosts And CBool(MySiteSettings.UseTokenForTimelines.Value)) Or (IsSavedPosts And CBool(MySiteSettings.UseTokenForSavedPosts.Value))) AndAlso + ' Not MySiteSettings.CredentialsExists Then + ' LogError(Nothing, "[429] You should use OAuth authorization or disable " & + ' IIf(IsSavedPosts, "token usage for downloading saved posts", "the use of token and cookies for downloading timelines")) + 'Else + ' LogError(Nothing, "Too many requests (429). Try again later!") + 'End If + 'MySiteSettings.SessionInterrupted = True + 'Throw New Plugin.ExitException With {.Silent = True} If ((Not IsSavedPosts And CBool(MySiteSettings.UseTokenForTimelines.Value)) Or (IsSavedPosts And CBool(MySiteSettings.UseTokenForSavedPosts.Value))) AndAlso Not MySiteSettings.CredentialsExists Then - LogError(Nothing, $"[{CInt(Responser.StatusCode)}] You should use OAuth authorization or disable " & + LogError(Nothing, "[429] You should use OAuth authorization or disable " & IIf(IsSavedPosts, "token usage for downloading saved posts", "the use of token and cookies for downloading timelines")) Else LogError(Nothing, "Too many requests (429). Try again later!") diff --git a/SCrawler/API/Redgifs/SiteSettings.vb b/SCrawler/API/Redgifs/SiteSettings.vb index 3fafd25..061e18f 100644 --- a/SCrawler/API/Redgifs/SiteSettings.vb +++ b/SCrawler/API/Redgifs/SiteSettings.vb @@ -22,6 +22,7 @@ Namespace API.RedGifs Friend ReadOnly Property Token As PropertyValue Private ReadOnly Property UserAgent As PropertyValue + Friend ReadOnly Property UseCookies As PropertyValue Friend ReadOnly Property TokenLastDateUpdated As PropertyValue Private Const TokenName As String = "authorization" #Region "TokenUpdateInterval" @@ -47,6 +48,7 @@ Namespace API.RedGifs End With Token = New PropertyValue(t, GetType(String), Sub(v) UpdateResponse(NameOf(Token), v)) UserAgent = New PropertyValue(If(Responser.UserAgentExists, Responser.UserAgent, String.Empty), GetType(String), Sub(v) UpdateResponse(NameOf(UserAgent), v)) + UseCookies = New PropertyValue(False) TokenLastDateUpdated = New PropertyValue(Now.AddYears(-1), GetType(Date)) TokenUpdateInterval = New PropertyValue(60 * 12, GetType(Integer)) TokenUpdateIntervalProvider = New TokenRefreshIntervalProvider diff --git a/SCrawler/API/Redgifs/UserData.vb b/SCrawler/API/Redgifs/UserData.vb index 333af81..426b4c2 100644 --- a/SCrawler/API/Redgifs/UserData.vb +++ b/SCrawler/API/Redgifs/UserData.vb @@ -36,6 +36,7 @@ Namespace API.RedGifs #End Region #Region "Download functions" Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken) + If Not MySettings.UseCookies.Value Then Responser.Cookies.Clear() DownloadData(1, Token) End Sub Private Overloads Sub DownloadData(ByVal Page As Integer, ByVal Token As CancellationToken) diff --git a/SCrawler/API/TikTok/UserData.vb b/SCrawler/API/TikTok/UserData.vb index 4040b05..4fffeba 100644 --- a/SCrawler/API/TikTok/UserData.vb +++ b/SCrawler/API/TikTok/UserData.vb @@ -422,7 +422,11 @@ Namespace API.TikTok End If If DateBefore.HasValue Then command &= $"--datebefore {DateBefore.Value.AddDays(1).ToStringDate(SimpleDateConverter)} " If DateAfter.HasValue Then command &= $"--dateafter {DateAfter.Value.AddDays(-1).ToStringDate(SimpleDateConverter)} " - If Not CBool(If(IsSingleObjectDownload, MySettings.UseParsedVideoDateSTD, MySettings.UseParsedVideoDate).Value) Then command &= "--no-mtime " + If Not CBool(If(IsSingleObjectDownload, MySettings.UseParsedVideoDateSTD, MySettings.UseParsedVideoDate).Value) Then + command &= "--no-mtime " + Else + command &= "--mtime " + End If If MySettings.CookiesNetscapeFile.Exists Then command &= $"--no-cookies-from-browser --cookies ""{MySettings.CookiesNetscapeFile}"" " command &= $"{URL} " If SupportOutput Then diff --git a/SCrawler/API/Twitter/EditorExchangeOptions.vb b/SCrawler/API/Twitter/EditorExchangeOptions.vb index 8dd7a89..23a71c4 100644 --- a/SCrawler/API/Twitter/EditorExchangeOptions.vb +++ b/SCrawler/API/Twitter/EditorExchangeOptions.vb @@ -50,6 +50,10 @@ Namespace API.Twitter Caption:="Force apply", ToolTip:="Force overrides the default parameters for the first download." & vbCr & "Applies to first download only.", LeftOffset:=DefaultOffset)> Friend Overridable Property DownloadModelForceApply As Boolean = False + + Friend Overridable Property LargeProfile As Boolean = False Private ReadOnly Property MySettings As Object Friend Sub New(ByVal s As SiteSettings) MyBase.New(s) @@ -76,6 +80,7 @@ Namespace API.Twitter UseMD5Comparison = u.UseMD5Comparison RemoveExistingDuplicates = u.RemoveExistingDuplicates MediaModelAllowNonUserTweets = u.MediaModelAllowNonUserTweets + LargeProfile = u.LargeProfile If Not TypeOf u Is Mastodon.UserData Then DownloadModelForceApply = u.DownloadModelForceApply DownloadBroadcasts = u.DownloadBroadcasts diff --git a/SCrawler/API/Twitter/SiteSettings.vb b/SCrawler/API/Twitter/SiteSettings.vb index 6d89c23..d4e59b6 100644 --- a/SCrawler/API/Twitter/SiteSettings.vb +++ b/SCrawler/API/Twitter/SiteSettings.vb @@ -38,6 +38,7 @@ Namespace API.Twitter Private Const CAT_DOWN As String = "Downloading" #End Region #Region "Auth" + Friend Property CookiesUpdateForce As Boolean = False Friend ReadOnly Property CookiesUpdate As PropertyValue @@ -45,9 +46,9 @@ Namespace API.Twitter Private ReadOnly Property UserAgentXML As PropertyValue - Friend ReadOnly Property UserAgent As String + Friend ReadOnly Property UserAgent(Optional ByVal Force As Boolean = False) As String Get - If CBool(UserAgentUse.Value) AndAlso Not CStr(UserAgentXML.Value).IsEmptyString Then + If (CBool(UserAgentUse.Value) Or Force) AndAlso Not CStr(UserAgentXML.Value).IsEmptyString Then Return UserAgentXML.Value Else Return String.Empty @@ -73,6 +74,7 @@ Namespace API.Twitter #Region "Limits" Friend Const TimerDisabled As Integer = -1 Friend Const TimerFirstUseTheSame As Integer = -2 + Friend Const TimerDefault As Integer = 20 Friend Property AbortOnLimit As PropertyValue @@ -143,6 +145,7 @@ Namespace API.Twitter End Property #End Region #Region "Initializer" + Private Const SettingsVersionCurrent As Integer = 1 Friend Sub New(ByVal AccName As String, ByVal Temp As Boolean) MyBase.New(TwitterSite, "x.com", AccName, Temp, My.Resources.SiteResources.TwitterIcon_32, My.Resources.SiteResources.TwitterIcon_32.ToBitmap) @@ -153,7 +156,7 @@ Namespace API.Twitter .Cookies.Changed = False End With - UseNewIconXML = New PropertyValue(False) + UseNewIconXML = New PropertyValue(True) CookiesUpdate = New PropertyValue(False) UserAgentUse = New PropertyValue(True) @@ -192,6 +195,10 @@ Namespace API.Twitter UseNetscapeCookies = True End Sub Friend Overrides Sub EndInit() + If Not SettingsVersion.Value = SettingsVersionCurrent Then + UseNewIconXML.Value = True + SettingsVersion.Value = SettingsVersionCurrent + End If UpdateIcon() MyBase.EndInit() End Sub @@ -223,7 +230,7 @@ Namespace API.Twitter End Sub Friend Overrides Sub DownloadDone(ByVal What As ISiteSettings.Download) If UserNumber > 0 Then - If CBool(CookiesUpdate.Value) Then + If CBool(CookiesUpdate.Value) Or CookiesUpdateForce Then With CookieKeeper.ParseNetscapeText(CookiesNetscapeFile.GetText(EDP.ReturnValue), EDP.ReturnValue) If .ListExists Then Responser.Cookies.Clear() @@ -250,6 +257,7 @@ Namespace API.Twitter End With End If LIMIT_ABORT = False + CookiesUpdateForce = False MyBase.DownloadDone(What) End Sub #End Region diff --git a/SCrawler/API/Twitter/UserData.vb b/SCrawler/API/Twitter/UserData.vb index 58b2599..329cae0 100644 --- a/SCrawler/API/Twitter/UserData.vb +++ b/SCrawler/API/Twitter/UserData.vb @@ -30,6 +30,7 @@ Namespace API.Twitter Private Const Name_GifsSpecialFolder As String = "GifsSpecialFolder" Private Const Name_GifsPrefix As String = "GifsPrefix" Private Const Name_IsCommunity As String = "IsCommunity" + Private Const Name_LargeProfile As String = "LargeProfile" Private Const Name_DownloadModelChanged As String = "DownloadModelChanged" #End Region #Region "Declarations" @@ -62,6 +63,47 @@ Namespace API.Twitter Friend Property GifsSpecialFolder As String = String.Empty Friend Property GifsPrefix As String = String.Empty Friend Property IsCommunity As Boolean = False +#Region "LargeProfile" + Friend Property LargeProfile As Boolean = False + Private ReadOnly Property LargeProfileOverride As Boolean + Get + Return LargeProfile And Not FirstDownloadComplete + End Get + End Property + Private ReadOnly Property CookiesUpdate As Boolean + Get + If LargeProfileOverride Then + MySettings.CookiesUpdateForce = True + Return True + Else + Return MySettings.CookiesUpdate.Value + End If + End Get + End Property + Private ReadOnly Property UserAgent As String + Get + If LargeProfileOverride Then + Return MySettings.UserAgent(True).IfNullOrEmpty(Settings.UserAgent) + Else + Return MySettings.UserAgent + End If + End Get + End Property + Private ReadOnly Property SleepTimerBeforeFirst As Integer + Get + Dim v% = MySettings.SleepTimerBeforeFirst.Value + If LargeProfileOverride And v <= 0 And v <> SiteSettings.TimerFirstUseTheSame Then v = SiteSettings.TimerFirstUseTheSame + Return v + End Get + End Property + Private ReadOnly Property SleepTimer As Integer + Get + Dim v% = MySettings.SleepTimer.Value + If LargeProfileOverride And v <= 0 Then v = SiteSettings.TimerDefault + Return v + End Get + End Property +#End Region Private ReadOnly LikesPosts As List(Of String) Private ReadOnly PostsKV As List(Of PKV) Private ReadOnly _DataNames As List(Of String) @@ -100,6 +142,7 @@ Namespace API.Twitter DownloadModelForceApply = .DownloadModelForceApply MediaModelAllowNonUserTweets = .MediaModelAllowNonUserTweets DownloadBroadcasts = .DownloadBroadcasts + LargeProfile = .LargeProfile Dim dModel As DownloadModels = DownloadModel If .DownloadModelMedia Then DownloadModel += DownloadModels.Media If .DownloadModelProfile Or .DownloadBroadcasts Then DownloadModel += DownloadModels.Profile @@ -155,6 +198,7 @@ Namespace API.Twitter StartMD5Checked = .Value(Name_StartMD5Checked).FromXML(Of Boolean)(False) MediaModelAllowNonUserTweets = .Value(Name_MediaModelAllowNonUserTweets).FromXML(Of Boolean)(False) IsCommunity = .Value(Name_IsCommunity).FromXML(Of Boolean)(False) + LargeProfile = .Value(Name_LargeProfile).FromXML(Of Boolean)(False) Else If Name.Contains("@") And Not IsCommunity Then IsCommunity = True @@ -180,6 +224,7 @@ Namespace API.Twitter .Add(Name_StartMD5Checked, StartMD5Checked.BoolToInteger) .Add(Name_MediaModelAllowNonUserTweets, MediaModelAllowNonUserTweets.BoolToInteger) .Add(Name_IsCommunity, IsCommunity.BoolToInteger) + .Add(Name_LargeProfile, LargeProfile.BoolToInteger) .Add(Name_TrueName, NameTrue(True)) End If End With @@ -615,6 +660,7 @@ nextpIndx: End If DownloadModelForceApply = False FirstDownloadComplete = True + LargeProfile = False Catch jsonNull_ex As JsonDocumentException When jsonNull_ex.State = WebDocumentEventArgs.States.Error Throw New Plugin.ExitException("No deserialized data found") Catch limit_ex As TwitterLimitException @@ -839,8 +885,8 @@ nextpIndx: End Class Private ReadOnly Property SleepTimerValue(ByVal First As Boolean) As Integer Get - Dim fTimer% = If(First, MySettings.SleepTimerBeforeFirst, MySettings.SleepTimer).Value - If First And fTimer = SiteSettings.TimerFirstUseTheSame Then fTimer = MySettings.SleepTimer.Value + Dim fTimer% = If(First, SleepTimerBeforeFirst, SleepTimer) + If First And fTimer = SiteSettings.TimerFirstUseTheSame Then fTimer = SleepTimer Return fTimer End Get End Property @@ -1060,10 +1106,10 @@ nextpIndx: Private Function GdlCreateConf(ByVal Path As SFile) As SFile Try Dim conf As SFile = $"{Path.PathWithSeparator}TwitterGdlConfig.conf" - Dim __userAgent$ = MySettings.UserAgent + Dim __userAgent$ = UserAgent If Not __userAgent.IsEmptyString Then __userAgent = $"""user-agent"": ""{__userAgent}""," Dim confText$ = "{""extractor"":{""cookies"": """ & MySettings.CookiesNetscapeFile.ToString.Replace("\", "/") & - $""",""cookies-update"": {IIf(CBool(MySettings.CookiesUpdate.Value), "true", "false")}," & __userAgent & + $""",""cookies-update"": {IIf(CookiesUpdate, "true", "false")}," & __userAgent & """twitter"":{""tweet-endpoint"": ""detail"",""cards"": false,""conversations"": true,""pinned"": false,""quoted"": false,""replies"": true,""retweets"": true,""strategy"": null,""text-tweets"": false,""twitpic"": false,""unique"": true,""users"": ""timeline"",""videos"": true}}}" If conf.Exists(SFO.Path, True, EDP.ThrowException) Then TextSaver.SaveTextToFile(confText, conf) If Not conf.Exists Then Throw New IO.FileNotFoundException("Can't find Twitter GDL config file", conf) diff --git a/SCrawler/Editors/GlobalSettingsForm.vb b/SCrawler/Editors/GlobalSettingsForm.vb index 8170215..20465da 100644 --- a/SCrawler/Editors/GlobalSettingsForm.vb +++ b/SCrawler/Editors/GlobalSettingsForm.vb @@ -187,24 +187,24 @@ Namespace Editors Private Sub MyDefs_ButtonOkClick(ByVal Sender As Object, ByVal e As KeyHandleEventArgs) Handles MyDefs.ButtonOkClick If MyDefs.MyFieldsChecker.AllParamsOK Then With Settings - Dim a As Func(Of String, Object, Integer) = - Function(t, v) MsgBoxE({$"You are set up higher than default count of along {t} downloading tasks." & vbNewLine & - $"Default: {SettingsCLS.DefaultMaxDownloadingTasks}" & vbNewLine & - $"Your value: {CInt(v)}" & vbNewLine & - "Increasing this value may lead to higher CPU usage." & vbNewLine & - "Do you really want to continue?", - "Increasing download tasks"}, - vbExclamation,,, {"Confirm", $"Set to default ({SettingsCLS.DefaultMaxDownloadingTasks})", "Cancel"}) + Dim a As Func(Of String, Integer, Object, Integer) = + Function(t, vc, v) MsgBoxE({$"You are set up higher than default count of along {t} downloading tasks." & vbNewLine & + $"Default: {vc}" & vbNewLine & + $"Your value: {CInt(v)}" & vbNewLine & + "Increasing this value may lead to higher CPU usage." & vbNewLine & + "Do you really want to continue?", + "Increasing download tasks"}, + vbExclamation,,, {"Confirm", $"Set to default ({vc})", "Cancel"}) If CInt(TXT_MAX_JOBS_USERS.Value) > SettingsCLS.DefaultMaxDownloadingTasks Then - Select Case a.Invoke("users", TXT_MAX_JOBS_USERS.Value) + Select Case a.Invoke("users", SettingsCLS.DefaultMaxDownloadingTasks, TXT_MAX_JOBS_USERS.Value) Case 1 : TXT_MAX_JOBS_USERS.Value = SettingsCLS.DefaultMaxDownloadingTasks Case 2 : Exit Sub End Select End If - If CInt(TXT_MAX_JOBS_CHANNELS.Value) > SettingsCLS.DefaultMaxDownloadingTasks Then - Select Case a.Invoke("channels", TXT_MAX_JOBS_CHANNELS.Value) - Case 1 : TXT_MAX_JOBS_CHANNELS.Value = SettingsCLS.DefaultMaxDownloadingTasks + If CInt(TXT_MAX_JOBS_CHANNELS.Value) > SettingsCLS.DefaultMaxDownloadingTasks_Channels Then + Select Case a.Invoke("channels", SettingsCLS.DefaultMaxDownloadingTasks_Channels, TXT_MAX_JOBS_CHANNELS.Value) + Case 1 : TXT_MAX_JOBS_CHANNELS.Value = SettingsCLS.DefaultMaxDownloadingTasks_Channels Case 2 : Exit Sub End Select End If @@ -407,7 +407,7 @@ Namespace Editors If Sender.DefaultButton = ADB.Refresh Then TXT_MAX_JOBS_USERS.Value = SettingsCLS.DefaultMaxDownloadingTasks End Sub Private Sub TXT_MAX_JOBS_CHANNELS_ActionOnButtonClick(ByVal Sender As ActionButton, ByVal e As EventArgs) Handles TXT_MAX_JOBS_CHANNELS.ActionOnButtonClick - If Sender.DefaultButton = ADB.Refresh Then TXT_MAX_JOBS_CHANNELS.Value = SettingsCLS.DefaultMaxDownloadingTasks + If Sender.DefaultButton = ADB.Refresh Then TXT_MAX_JOBS_CHANNELS.Value = SettingsCLS.DefaultMaxDownloadingTasks_Channels End Sub Private Sub ChangePositionControlsEnabling() Handles OPT_FILE_NAME_REPLACE.CheckedChanged, OPT_FILE_NAME_ADD_DATE.CheckedChanged Dim b As Boolean = OPT_FILE_NAME_ADD_DATE.Checked And OPT_FILE_NAME_ADD_DATE.Enabled diff --git a/SCrawler/My Project/AssemblyInfo.vb b/SCrawler/My Project/AssemblyInfo.vb index d4c040d..a90cd29 100644 --- a/SCrawler/My Project/AssemblyInfo.vb +++ b/SCrawler/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + + diff --git a/SCrawler/SettingsCLS.vb b/SCrawler/SettingsCLS.vb index 84b1545..c119f71 100644 --- a/SCrawler/SettingsCLS.vb +++ b/SCrawler/SettingsCLS.vb @@ -23,6 +23,7 @@ Imports DoubleClickBehavior = SCrawler.DownloadObjects.STDownloader.DoubleClickB Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable #Region "Constants: defaults" Friend Const DefaultMaxDownloadingTasks As Integer = 5 + Friend Const DefaultMaxDownloadingTasks_Channels As Integer = 1 Friend Const TaskStackNamePornSite As String = "Porn sites" Friend Const Name_Node_Sites As String = "Sites" Private Const SitesValuesSeparator As String = "," @@ -194,7 +195,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable Private ReadOnly BlackListFile As SFile = $"{SettingsFolderName}\BlackList.txt" Private ReadOnly UsersSettingsFile As SFile = $"{SettingsFolderName}\Users.xml" Private ReadOnly Property SettingsVersion As XMLValue(Of Integer) - Private Const SettingsVersionCurrent As Integer = 2 + Private Const SettingsVersionCurrent As Integer = 3 Friend ShortcutOpenFeed As New ButtonKey(Keys.F, True) Friend ShortcutOpenSearch As New ButtonKey(Keys.F,, True) Private Sub ChangeFeedOpenMode() @@ -366,9 +367,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable ReparseMissingInTheRoutine = New XMLValue(Of Boolean)("ReparseMissingInTheRoutine", False, MyXML, n) UseDefaultAccountIfMissing = New XMLValue(Of Boolean)("UseDefaultAccountIfMissing", True, MyXML, n) AutomationBrushUndownloadedPlansMinutes = New XMLValue(Of Integer)("AutomationBrushUndownloadedPlansMinutes", 10080, MyXML, n) - DownDetectorEnabled = New XMLValue(Of Boolean)("DownDetectorEnabled", True, MyXML, n) - 'TODELETE: DownDetectorEnabled change - If SettingsVersion.Value < SettingsVersionCurrent Then DownDetectorEnabled.Value = False 'SettingsVersionCurrent = 2 + DownDetectorEnabled = New XMLValue(Of Boolean)("DownDetectorEnabled", False, MyXML, n) 'Downloading: file naming n = {"Downloading", "FileName"} @@ -392,7 +391,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable ChannelsDefaultReadyForDownload = New XMLValue(Of Boolean)("ChannelsDefaultReadyForDownload", False, MyXML, n) ChannelsDefaultTemporary = New XMLValue(Of Boolean)("ChannelsDefaultTemporary", True, MyXML, n) ChannelsHideExistsUser = New XMLValue(Of Boolean)("HideExistsUser", True, MyXML, n) - ChannelsMaxJobsCount = New XMLValue(Of Integer)("MaxJobsCount", DefaultMaxDownloadingTasks, MyXML, n) + ChannelsMaxJobsCount = New XMLValue(Of Integer)("MaxJobsCount", DefaultMaxDownloadingTasks_Channels, MyXML, n) n = {Name_Node_Sites, "Channels", "Users"} FromChannelDownloadTop = New XMLValue(Of Integer)("FromChannelDownloadTop", 10, MyXML, n) FromChannelDownloadTopUse = New XMLValue(Of Boolean)("FromChannelDownloadTopUse", False, MyXML, n) @@ -497,6 +496,8 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable AdvancedFilter.IsViewFilter = True Labels.AddRange({AdvancedFilter}.GetGroupsLabels, False) + 'TODELETE: DefaultMaxDownloadingTasks_Channels + If Not SettingsVersion = SettingsVersionCurrent Then ChannelsMaxJobsCount.Value = DefaultMaxDownloadingTasks_Channels 'SettingsVersionCurrent = 3 SettingsVersion.Value = SettingsVersionCurrent MyXML.EndUpdate()