2025.8.1.0

YT Update 'ReplaceModificationDate' SCrawler API.Instagram: fix 'LastCursor' issue API.Reddit: add OAuth validation; add default credentials; hide unused controls; add 'SeparatedTasks'; bypass 429 error; fix crossposts downloading API.Redgifs: force delete cookies if user added them API.TikTok: yt-dlp modification (date change) API.Twitter: simplify large profiles download SettingsCLS: change default max value for channel downloads
2026-03-14 15:52:18 +00:00 · 2025-08-01 21:49:40 +03:00
parent 05772a9fc4
commit e09752a2d5
16 changed files with 239 additions and 70 deletions
--- a/Changelog.md
+++ b/Changelog.md
@@ -1,12 +1,26 @@
+# 2025.8.1.0
+
+*2025-08-01*
+
+- Added
+  - Sites:
+    - Reddit: **bypass error `429`**
+    - Twitter: **[large profile option](https://github.com/AAndyProgram/SCrawler/wiki/Settings#twitter-user-settings) in user settings**
+  - Minor improvements
+- Updated
+  - yt-dlp up to version **2025.27.21**
+  - gallery-dl up to version **1.30.2**
+- Fixed
+  - Reddit: in some cases crossposts don't download
+  - Minor bugs
+
 # 2025.7.18.0

 *2025-07-18*

 - Added
  - Sites:
-    - OnlyFans:
-      - **bypass unpurchased videos**
-      - support for GIF files
+    - OnlyFans: support for GIF files
    - Reddit: extended `429` error handling
    - Xhamster: support for downloading 'moments'
  - Minor improvements
@@ -14,6 +28,7 @@
  - yt-dlp up to version **2025.06.30**
  - gallery-dl up to version **1.30.0**
 - Fixed
+  - OnlyFans: **hanging on purchased content**
  - Minor bugs

 # 2025.6.12.0
--- a/Project/AssemblyInfo.vb
+++ b/Project/AssemblyInfo.vb
@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>

-<Assembly: AssemblyVersion("2025.6.12.0")>
-<Assembly: AssemblyFileVersion("2025.6.12.0")>
+<Assembly: AssemblyVersion("2025.8.1.0")>
+<Assembly: AssemblyFileVersion("2025.8.1.0")>
 <Assembly: NeutralResourcesLanguage("en")>
--- a/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb
+++ b/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb
@@ -823,7 +823,9 @@ Namespace API.YouTube.Objects
                        'cmd = $"yt-dlp -f ""{cmd}"""
                        'cmd = $"yt-dlp -f {cmd}"
                        cmd = $"{YTDLP_NAME} -f {cmd}"
-                        If Not MyYouTubeSettings.ReplaceModificationDate Then cmd &= " --no-mtime"
+                        'yt-dlp 2025.07.21
+                        'If Not MyYouTubeSettings.ReplaceModificationDate Then cmd &= " --no-mtime"
+                        cmd &= $" --{IIf(MyYouTubeSettings.ReplaceModificationDate.Value, String.Empty, "no-")}mtime"
                        cmd.StringAppend(formats, " ")
                        cmd.StringAppend(subs, " ")
                        cmd.StringAppend(YouTubeFunctions.GetCookiesCommand(WithCookies, YouTubeCookieNetscapeFile), " ")
--- a/SCrawler.YouTubeDownloader/My
+++ b/SCrawler.YouTubeDownloader/My
@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>

-<Assembly: AssemblyVersion("2025.6.12.0")>
-<Assembly: AssemblyFileVersion("2025.6.12.0")>
+<Assembly: AssemblyVersion("2025.8.1.0")>
+<Assembly: AssemblyFileVersion("2025.8.1.0")>
 <Assembly: NeutralResourcesLanguage("en")>
--- a/SCrawler/API/Instagram/UserData.vb
+++ b/SCrawler/API/Instagram/UserData.vb
@@ -412,6 +412,7 @@ Namespace API.Instagram
                ThrowAny(Token)
                HasError = False
                Dim dt As Func(Of Boolean) = Function() (CBool(MySiteSettings.DownloadTimeline.Value) And GetTimeline) Or IsSavedPosts
+                If FirstLoadingDone Then LastCursor = String.Empty
                If dt.Invoke And Not LastCursor.IsEmptyString Then
                    s = IIf(IsSavedPosts, Sections.SavedPosts, Sections.Timeline)
                    upClaimRequest.Invoke
--- a/SCrawler/API/Reddit/SiteSettings.vb
+++ b/SCrawler/API/Reddit/SiteSettings.vb
@@ -15,21 +15,32 @@ Imports PersonalUtilities.Tools.Web.Clients.Base
 Imports PersonalUtilities.Tools.Web.Documents.JSON
 Imports PersonalUtilities.Functions.XML
 Imports PersonalUtilities.Functions.RegularExpressions
+Imports DN = SCrawler.API.Base.DeclaredNames
 Imports DownDetector = SCrawler.API.Base.DownDetector
 Imports Download = SCrawler.Plugin.ISiteSettings.Download
 Namespace API.Reddit
-    <Manifest(RedditSiteKey), SavedPosts, SpecialForm(False), UseDownDetector>
+    <Manifest(RedditSiteKey), SavedPosts, SeparatedTasks, SpecialForm(False), UseDownDetector>
    Friend Class SiteSettings : Inherits SiteSettingsBase : Implements DownDetector.IDownDetector
 #Region "Declarations"
 #Region "Authorization"
+        Private Const ApiClientID_Default As String = "dYctRA-SIJxyykHe27lGZg"
+        Private Const ApiClientSecret_Default As String = "_5D6KzplRPDga-es1YlpzDIe9hiFlg"
        <PropertyOption(ControlText:="Login", ControlToolTip:="Your authorization username", IsAuth:=True), PXML, PClonable(Clone:=False)>
        Friend ReadOnly Property AuthUserName As PropertyValue
        <PropertyOption(ControlText:="Password", ControlToolTip:="Your authorization password", IsAuth:=True), PXML, PClonable(Clone:=False)>
        Friend ReadOnly Property AuthPassword As PropertyValue
        <PropertyOption(ControlText:="Client ID", ControlToolTip:="Your registered app client ID", IsAuth:=True), PXML, PClonable(Clone:=False)>
        Friend ReadOnly Property ApiClientID As PropertyValue
+        <PropertyUpdater(NameOf(ApiClientID))> Private Function ApiClientID_SetDefault() As Boolean
+            ApiClientID.Value = ApiClientID_Default
+            Return True
+        End Function
        <PropertyOption(ControlText:="Client Secret", ControlToolTip:="Your registered app client secret", IsAuth:=True), PXML, PClonable(Clone:=False)>
        Friend ReadOnly Property ApiClientSecret As PropertyValue
+        <PropertyUpdater(NameOf(ApiClientSecret))> Private Function ApiClientSecret_SetDefault() As Boolean
+            ApiClientSecret.Value = ApiClientSecret_Default
+            Return True
+        End Function
        <PropertyOption(ControlText:="Bearer token",
                        ControlToolTip:="Bearer token (can be null)." & vbCr &
                                        "If you are using cookies to download the timeline, it is highly recommended that you add a token." & vbCr &
@@ -103,12 +114,15 @@ Namespace API.Reddit
        End Function
 #End Region
 #Region "Other"
-        <PropertyOption(ControlText:="Use M3U8", ControlToolTip:="Use M3U8 or mp4 for Reddit videos", IsAuth:=False), PXML, PClonable>
+        <PropertyOption(ControlText:="Use M3U8", ControlToolTip:="Use M3U8 or mp4 for Reddit videos"), PXML, PClonable, HiddenControl>
        Friend ReadOnly Property UseM3U8 As PropertyValue
-        <PropertyOption(ControlText:="Check image", ControlToolTip:="Check the image if it exists before downloading (it makes downloading very slow)", IsAuth:=False), PXML, PClonable>
+        <PropertyOption(ControlText:="Check image", ControlToolTip:="Check the image if it exists before downloading (it makes downloading very slow)"), PXML, PClonable, HiddenControl>
        Friend ReadOnly Property CheckImage As PropertyValue
-        <PropertyOption(ControlText:="Check image: get original", ControlToolTip:="Get the original image if it exists", IsAuth:=False), PXML, PClonable>
+        <PropertyOption(ControlText:="Check image: get original", ControlToolTip:="Get the original image if it exists"), PXML, PClonable, HiddenControl>
        Friend ReadOnly Property CheckImageReturnOrig As PropertyValue
+        <PropertyOption(ControlText:=DN.ConcurrentDownloadsCaption,
+                        ControlToolTip:=DN.ConcurrentDownloadsToolTip, AllowNull:=False), PXML, TaskCounter, PClonable>
+        Friend ReadOnly Property ConcurrentDownloads As PropertyValue
 #End Region
 #Region "IDownDetector Support"
        Private ReadOnly Property IDownDetector_Value As Integer Implements DownDetector.IDownDetector.Value
@@ -160,6 +174,7 @@ Namespace API.Reddit
            UseM3U8 = New PropertyValue(True)
            CheckImage = New PropertyValue(False)
            CheckImageReturnOrig = New PropertyValue(True)
+            ConcurrentDownloads = New PropertyValue(1)

            MDD = New MyDownDetector(Me)

@@ -167,10 +182,13 @@ Namespace API.Reddit
            ImageVideoContains = "reddit.com"
            UserRegex = RParams.DM("[htps:/]{7,8}.*?reddit.com/([user]{1,4})/([^/\?&]+)", 0, RegexReturn.ListByMatch, EDP.ReturnValue)
        End Sub
-        Private Const SettingsVersionCurrent As Integer = 2
+        Private Const SettingsVersionCurrent As Integer = 3
        Friend Overrides Sub EndInit()
            If CInt(SettingsVersion.Value) < SettingsVersionCurrent Then
                SettingsVersion.Value = SettingsVersionCurrent
+                UseM3U8.Value = True
+                CheckImage.Value = False
+                CheckImageReturnOrig.Value = True
                BearerTokenUseCurl.Value = False
            End If
            MyBase.EndInit()
@@ -208,6 +226,7 @@ Namespace API.Reddit
            End Sub
        End Class
        Friend Property SessionInterrupted As Boolean = False
+        Friend Property RequestCount As Integer = 0
        Friend Overrides Function ReadyToDownload(ByVal What As Download) As Boolean
            If What = Download.Main Then
                Return Not SessionInterrupted
@@ -223,6 +242,7 @@ Namespace API.Reddit
        End Function
        Friend Overrides Sub DownloadDone(ByVal What As Download)
            SessionInterrupted = False
+            RequestCount = 0
            MDD.Reset()
            MyBase.DownloadDone(What)
        End Sub
--- a/SCrawler/API/Reddit/UserData.vb
+++ b/SCrawler/API/Reddit/UserData.vb
@@ -8,19 +8,20 @@
 ' but WITHOUT ANY WARRANTY
 Imports System.Net
 Imports System.Threading
+Imports PersonalUtilities.Functions.RegularExpressions
+Imports PersonalUtilities.Functions.XML
+Imports PersonalUtilities.Tools.ImageRenderer
+Imports PersonalUtilities.Tools.Web.Clients
+Imports PersonalUtilities.Tools.Web.Clients.Base
+Imports PersonalUtilities.Tools.Web.Documents.JSON
 Imports SCrawler.API.Base
 Imports SCrawler.API.Reddit.RedditViewExchange
 Imports SCrawler.API.YouTube.Objects
 Imports SCrawler.Plugin.Hosts
-Imports PersonalUtilities.Functions.XML
-Imports PersonalUtilities.Functions.RegularExpressions
-Imports PersonalUtilities.Tools.ImageRenderer
-Imports PersonalUtilities.Tools.Web.Clients
-Imports PersonalUtilities.Tools.Web.Documents.JSON
+Imports CPeriod = SCrawler.API.Reddit.IRedditView.Period
+Imports CView = SCrawler.API.Reddit.IRedditView.View
 Imports UStates = SCrawler.API.Base.UserMedia.States
 Imports UTypes = SCrawler.API.Base.UserMedia.Types
-Imports CView = SCrawler.API.Reddit.IRedditView.View
-Imports CPeriod = SCrawler.API.Reddit.IRedditView.Period
 Namespace API.Reddit
    Friend Class UserData : Inherits UserDataBase : Implements IChannelLimits, IRedditView
 #Region "Declarations"
@@ -269,6 +270,8 @@ Namespace API.Reddit
                End If
            End With

+            Responser.ProcessExceptionDecision = AddressOf Err429Process
+
            _TotalPostsDownloaded = 0
            If IsSavedPosts Then
                Responser.DecodersError = EDP.ReturnValue
@@ -304,6 +307,7 @@ Namespace API.Reddit
 #End Region
 #Region "Download Functions (User, Channel)"
        Private Err429Count As Integer = 0
+        Private Err429TryAgain As Boolean = False
        Private _TotalPostsDownloaded As Integer = 0
        Private ReadOnly _CrossPosts As List(Of String)
        Private Const SiteGfycatKey As String = "gfycat"
@@ -311,6 +315,28 @@ Namespace API.Reddit
        Private Const Node_CrosspostRootId As String = "crosspostRootId"
        Private Const Node_CrosspostParentId As String = "crosspostParentId"
        Private Const Node_CrosspostParent As String = "crosspost_parent"
+        Private Sub Wait429()
+            With MySiteSettings
+                If Not Err429TryAgain Then .RequestCount += 1
+                Err429TryAgain = False
+                If (.RequestCount Mod 100) = 0 Then Thread.Sleep(60100)
+            End With
+        End Sub
+        Private Function Err429Process(ByVal Status As IResponserStatus, ByVal NullArg As Object, ByVal CurrErr As ErrorsDescriber) As ErrorsDescriber
+            If Not Status Is Nothing AndAlso Status.StatusCode = 429 Then
+                If Err429Count = 0 Then
+                    Err429Count += 1
+                    MySiteSettings.RequestCount = 100
+                    Err429TryAgain = True
+                    Return EDP.ReturnValue
+                End If
+            End If
+            Return CurrErr
+        End Function
+        Private Sub Err429Reset()
+            Err429Count = 0
+            Err429TryAgain = False
+        End Sub
        Private Sub DownloadDataUser(ByVal POST As String, ByVal Token As CancellationToken)
            Dim eObj% = 0
            Dim round% = 0
@@ -331,8 +357,10 @@ Namespace API.Reddit
                    'URL = $"https://gateway.reddit.com/desktopapi/v1/user/{NameTrue}/posts?rtj=only&allow_quarantined=true&allow_over18=1&include=identity&after={POST}&dist=25&sort={View}&t={Period}&layout=classic"
                    URL = $"https://oauth.reddit.com/user/{NameTrue}/submitted.json?rtj=only&allow_quarantined=true&allow_over18=1&include=identity&after={POST}&dist=25&sort={View}&t={Period}&layout=classic"
                    ThrowAny(Token)
+                    Wait429()
                    Dim r$ = Responser.GetResponse(URL)
                    If Not r.IsEmptyString Then
+                        Err429Reset()
                        Using w As EContainer = JsonDocument.Parse(r).XmlIfNothing
                            If w.Count > 0 Then
                                'n = w.GetNode(JsonNodesJson)
@@ -347,6 +375,7 @@ Namespace API.Reddit
                                                If CheckNode(.Self) Then

                                                    'Obtain post ID
+                                                    PostID = String.Empty
                                                    PostTmp = .Value("name") '.Name
                                                    If PostTmp.IsEmptyString Then PostTmp = .Value("id")
                                                    If PostTmp.IsEmptyString Then Continue For
@@ -354,8 +383,9 @@ Namespace API.Reddit
                                                    If IsCrossPost(.Self) Then
                                                        _CrossPosts.ListAddList({ .Value(Node_CrosspostRootId),
                                                                                  .Value(Node_CrosspostParentId),
-                                                                                  .Value(Node_CrosspostParent)}, LNC)
-                                                        Continue For
+                                                                                  .Value(Node_CrosspostParent),
+                                                                                  PostTmp}, LNC)
+                                                        If ParseUserMediaOnly Then Continue For
                                                    Else
                                                        If Not _CrossPosts.Contains(PostTmp) Then PostID = PostTmp : PostTmp = String.Empty
                                                    End If
@@ -384,6 +414,8 @@ Namespace API.Reddit
                        End Using
                        If POST.IsEmptyString And ExistsDetected Then Exit Sub
                        If Not _PostID().IsEmptyString And NewPostDetected Then DownloadDataUser(_PostID(), Token)
+                    ElseIf Err429TryAgain Then
+                        Continue Do
                    End If
                    _completed = True
                Catch ex As Exception
@@ -420,9 +452,11 @@ Namespace API.Reddit
                    End If

                    ThrowAny(Token)
+                    Wait429()
                    Dim r$ = Responser.GetResponse(URL)
-                    If IsSavedPosts Then Err429Count = 0
+                    'If IsSavedPosts Then Err429Count = 0
                    If Not r.IsEmptyString Then
+                        Err429Reset()
                        Using w As EContainer = JsonDocument.Parse(r).XmlIfNothing
                            If w.Count > 0 Then
                                n = w.GetNode(ChannelJsonNodes)
@@ -479,6 +513,8 @@ Namespace API.Reddit
                        End Using
                        If POST.IsEmptyString And ExistsDetected Then Exit Sub
                        If Not PostID.IsEmptyString And NewPostDetected Then DownloadDataChannel(PostID, Token)
+                    ElseIf Err429TryAgain Then
+                        Continue Do
                    End If
                    _completed = True
                Catch ex As Exception
@@ -496,11 +532,13 @@ Namespace API.Reddit
        End Sub
 #End Region
 #Region "GetUserInfo"
-        Private Sub GetUserInfo()
+        Private Sub GetUserInfo(Optional ByVal Round As Integer = 0)
            Try
                If Not IsSavedPosts And ChannelInfo Is Nothing Then
+                    Wait429()
                    Dim r$ = Responser.GetResponse($"https://reddit.com/{IIf(IsChannel, "r", "user")}/{NameTrue}/about.json",, EDP.ReturnValue)
                    If Not r.IsEmptyString Then
+                        Err429Reset()
                        Using j As EContainer = JsonDocument.Parse(r)
                            If Not j Is Nothing AndAlso j.Contains({"data", "subreddit"}) Then
                                If ID.IsEmptyString Then ID = j.Value({"data"}, "id")
@@ -516,6 +554,8 @@ Namespace API.Reddit
                                End With
                            End If
                        End Using
+                    ElseIf Err429TryAgain And Round < 2 Then
+                        GetUserInfo(Round + 1)
                    End If
                End If
            Catch ex As Exception
@@ -631,16 +671,21 @@ Namespace API.Reddit
                        Else
                            Dim tPostId$ = e.Value(Node_CrosspostParent).IfNullOrEmpty(e.Value(Node_CrosspostParentId)).IfNullOrEmpty(e.Value(Node_CrosspostRootId))
                            If Not PostID.IsEmptyString Then
-                                Dim r$ = Responser.GetResponse($"https://www.reddit.com/comments/{tPostId.Split("_").LastOrDefault}/.json",, EDP.ReturnValue)
-                                If Not r.IsEmptyString Then
-                                    Using j As EContainer = JsonDocument.Parse(r, EDP.ReturnValue)
-                                        If j.ListExists Then
-                                            With j.ItemF({0, "data", "children", 0, "data"})
-                                                If .ListExists Then added = ParseContainer(.Self, PostID, PostDate, UserID, False, PostText)
-                                            End With
-                                        End If
-                                    End Using
-                                End If
+                                For ri% = 0 To 1
+                                    Wait429()
+                                    Dim r$ = Responser.GetResponse($"https://www.reddit.com/comments/{tPostId.Split("_").LastOrDefault}/.json",, EDP.ReturnValue)
+                                    If Not r.IsEmptyString Then
+                                        Err429Reset()
+                                        Using j As EContainer = JsonDocument.Parse(r, EDP.ReturnValue)
+                                            If j.ListExists Then
+                                                With j.ItemF({0, "data", "children", 0, "data"})
+                                                    If .ListExists Then added = ParseContainer(.Self, PostID, PostDate, UserID, False, PostText)
+                                                End With
+                                            End If
+                                        End Using
+                                        Exit For
+                                    End If
+                                Next
                            End If
                        End If
                    End If
@@ -906,7 +951,10 @@ Namespace API.Reddit
                                        End If
                                        Continue For
                                    Else
+                                        Wait429()
                                        r = Responser.GetResponse(m.URL,, e)
+                                        If r.IsEmptyString And Err429TryAgain Then _repeatForRedgifs = True
+                                        If Not r.IsEmptyString Then Err429Reset()
                                    End If
                                Loop While _repeatForRedgifs
                            Else
@@ -944,11 +992,13 @@ Namespace API.Reddit
                    RedGifsResponser = RedGifsHost.Responser.Copy
                    Dim respNoHeaders As Responser = Responser.Copy
                    Dim m As UserMedia, m2 As UserMedia
-                    Dim r$, url$
+                    Dim r$ = String.Empty, url$
+                    Dim ri As Byte
                    Dim j As EContainer
                    Dim lastCount%, li%
                    Dim rv As New ErrorsDescriber(EDP.ReturnValue)
                    respNoHeaders.Headers.Clear()
+                    respNoHeaders.ProcessExceptionDecision = AddressOf Err429Process
                    ProgressPre.ChangeMax(_ContentList.Count)
                    For i% = 0 To _ContentList.Count - 1
                        m = _ContentList(i)
@@ -956,9 +1006,14 @@ Namespace API.Reddit
                        If m.State = UStates.Missing AndAlso Not m.Post.ID.IsEmptyString Then
                            ThrowAny(Token)
                            url = $"https://www.reddit.com/comments/{m.Post.ID.Split("_").LastOrDefault}/.json"
-                            r = Responser.GetResponse(url,, rv)
-                            If r.IsEmptyString Then r = respNoHeaders.GetResponse(url,, rv)
+                            For ri = 0 To 1
+                                Wait429()
+                                r = Responser.GetResponse(url,, rv)
+                                If r.IsEmptyString Then Wait429() : r = respNoHeaders.GetResponse(url,, rv)
+                                If Not (r.IsEmptyString And Err429TryAgain) Then Exit For
+                            Next
                            If Not r.IsEmptyString Then
+                                Err429Reset()
                                j = JsonDocument.Parse(r, rv)
                                If Not j Is Nothing Then
                                    If j.Count > 0 Then
@@ -1101,13 +1156,22 @@ Namespace API.Reddit
                ElseIf .StatusCode = HttpStatusCode.InternalServerError Then '500
                    If Not IsNothing(EObj) AndAlso IsNumeric(EObj) AndAlso CInt(EObj) = HttpStatusCode.InternalServerError Then Return 1
                    Return HttpStatusCode.InternalServerError
-                ElseIf .StatusCode = 429 And IsSavedPosts And Err429Count = 0 Then '429 (saved)
-                    Err429Count += 1
-                    Return 429
+                    'ElseIf .StatusCode = 429 And IsSavedPosts And Err429Count = 0 Then '429 (saved)
+                    '    Err429Count += 1
+                    '    Return 429
                ElseIf .StatusCode = 429 Then '429 (all)
+                    'If ((Not IsSavedPosts And CBool(MySiteSettings.UseTokenForTimelines.Value)) Or (IsSavedPosts And CBool(MySiteSettings.UseTokenForSavedPosts.Value))) AndAlso
+                    '   Not MySiteSettings.CredentialsExists Then
+                    '    LogError(Nothing, "[429] You should use OAuth authorization or disable " &
+                    '                      IIf(IsSavedPosts, "token usage for downloading saved posts", "the use of token and cookies for downloading timelines"))
+                    'Else
+                    '    LogError(Nothing, "Too many requests (429). Try again later!")
+                    'End If
+                    'MySiteSettings.SessionInterrupted = True
+                    'Throw New Plugin.ExitException With {.Silent = True}
                    If ((Not IsSavedPosts And CBool(MySiteSettings.UseTokenForTimelines.Value)) Or (IsSavedPosts And CBool(MySiteSettings.UseTokenForSavedPosts.Value))) AndAlso
                       Not MySiteSettings.CredentialsExists Then
-                        LogError(Nothing, $"[{CInt(Responser.StatusCode)}] You should use OAuth authorization or disable " &
+                        LogError(Nothing, "[429] You should use OAuth authorization or disable " &
                                          IIf(IsSavedPosts, "token usage for downloading saved posts", "the use of token and cookies for downloading timelines"))
                    Else
                        LogError(Nothing, "Too many requests (429). Try again later!")
--- a/SCrawler/API/Redgifs/SiteSettings.vb
+++ b/SCrawler/API/Redgifs/SiteSettings.vb
@@ -22,6 +22,7 @@ Namespace API.RedGifs
        Friend ReadOnly Property Token As PropertyValue
        <PropertyOption, ControlNumber(2), PClonable, HiddenControl>
        Private ReadOnly Property UserAgent As PropertyValue
+        <PXML> Friend ReadOnly Property UseCookies As PropertyValue
        <PXML> Friend ReadOnly Property TokenLastDateUpdated As PropertyValue
        Private Const TokenName As String = "authorization"
 #Region "TokenUpdateInterval"
@@ -47,6 +48,7 @@ Namespace API.RedGifs
            End With
            Token = New PropertyValue(t, GetType(String), Sub(v) UpdateResponse(NameOf(Token), v))
            UserAgent = New PropertyValue(If(Responser.UserAgentExists, Responser.UserAgent, String.Empty), GetType(String), Sub(v) UpdateResponse(NameOf(UserAgent), v))
+            UseCookies = New PropertyValue(False)
            TokenLastDateUpdated = New PropertyValue(Now.AddYears(-1), GetType(Date))
            TokenUpdateInterval = New PropertyValue(60 * 12, GetType(Integer))
            TokenUpdateIntervalProvider = New TokenRefreshIntervalProvider
--- a/SCrawler/API/Redgifs/UserData.vb
+++ b/SCrawler/API/Redgifs/UserData.vb
@@ -36,6 +36,7 @@ Namespace API.RedGifs
 #End Region
 #Region "Download functions"
        Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
+            If Not MySettings.UseCookies.Value Then Responser.Cookies.Clear()
            DownloadData(1, Token)
        End Sub
        Private Overloads Sub DownloadData(ByVal Page As Integer, ByVal Token As CancellationToken)
--- a/SCrawler/API/TikTok/UserData.vb
+++ b/SCrawler/API/TikTok/UserData.vb
@@ -422,7 +422,11 @@ Namespace API.TikTok
            End If
            If DateBefore.HasValue Then command &= $"--datebefore {DateBefore.Value.AddDays(1).ToStringDate(SimpleDateConverter)} "
            If DateAfter.HasValue Then command &= $"--dateafter {DateAfter.Value.AddDays(-1).ToStringDate(SimpleDateConverter)} "
-            If Not CBool(If(IsSingleObjectDownload, MySettings.UseParsedVideoDateSTD, MySettings.UseParsedVideoDate).Value) Then command &= "--no-mtime "
+            If Not CBool(If(IsSingleObjectDownload, MySettings.UseParsedVideoDateSTD, MySettings.UseParsedVideoDate).Value) Then
+                command &= "--no-mtime "
+            Else
+                command &= "--mtime "
+            End If
            If MySettings.CookiesNetscapeFile.Exists Then command &= $"--no-cookies-from-browser --cookies ""{MySettings.CookiesNetscapeFile}"" "
            command &= $"{URL} "
            If SupportOutput Then
--- a/SCrawler/API/Twitter/EditorExchangeOptions.vb
+++ b/SCrawler/API/Twitter/EditorExchangeOptions.vb
@@ -50,6 +50,10 @@ Namespace API.Twitter
                  Caption:="Force apply",
                  ToolTip:="Force overrides the default parameters for the first download." & vbCr & "Applies to first download only.", LeftOffset:=DefaultOffset)>
        Friend Overridable Property DownloadModelForceApply As Boolean = False
+        <PSetting(Address:=SettingAddress.User,
+                  Caption:="Large profile",
+                  ToolTip:="This setting is only used on the first download and is intended to temporarily override the default site settings if they are incompatible with downloading large profiles. After the first download is complete, this option will be disabled and cannot be enabled again.")>
+        Friend Overridable Property LargeProfile As Boolean = False
        Private ReadOnly Property MySettings As Object
        Friend Sub New(ByVal s As SiteSettings)
            MyBase.New(s)
@@ -76,6 +80,7 @@ Namespace API.Twitter
            UseMD5Comparison = u.UseMD5Comparison
            RemoveExistingDuplicates = u.RemoveExistingDuplicates
            MediaModelAllowNonUserTweets = u.MediaModelAllowNonUserTweets
+            LargeProfile = u.LargeProfile
            If Not TypeOf u Is Mastodon.UserData Then
                DownloadModelForceApply = u.DownloadModelForceApply
                DownloadBroadcasts = u.DownloadBroadcasts
--- a/SCrawler/API/Twitter/SiteSettings.vb
+++ b/SCrawler/API/Twitter/SiteSettings.vb
@@ -38,6 +38,7 @@ Namespace API.Twitter
        Private Const CAT_DOWN As String = "Downloading"
 #End Region
 #Region "Auth"
+        Friend Property CookiesUpdateForce As Boolean = False
        <PropertyOption(ControlText:="Update cookies", ControlToolTip:="Update cookies during requests", IsAuth:=True), PXML, PClonable, HiddenControl>
        Friend ReadOnly Property CookiesUpdate As PropertyValue
        <PropertyOption(ControlText:="Use UserAgent", ControlToolTip:="Use UserAgent in requests", IsAuth:=True), PXML, PClonable>
@@ -45,9 +46,9 @@ Namespace API.Twitter
        <PropertyOption(ControlText:="UserAgent", IsAuth:=True, AllowNull:=True, InheritanceName:=SettingsCLS.HEADER_DEF_UserAgent),
         PXML("UserAgent", OnlyForChecked:=True), PClonable>
        Private ReadOnly Property UserAgentXML As PropertyValue
-        Friend ReadOnly Property UserAgent As String
+        Friend ReadOnly Property UserAgent(Optional ByVal Force As Boolean = False) As String
            Get
-                If CBool(UserAgentUse.Value) AndAlso Not CStr(UserAgentXML.Value).IsEmptyString Then
+                If (CBool(UserAgentUse.Value) Or Force) AndAlso Not CStr(UserAgentXML.Value).IsEmptyString Then
                    Return UserAgentXML.Value
                Else
                    Return String.Empty
@@ -73,6 +74,7 @@ Namespace API.Twitter
 #Region "Limits"
        Friend Const TimerDisabled As Integer = -1
        Friend Const TimerFirstUseTheSame As Integer = -2
+        Friend Const TimerDefault As Integer = 20
        <PropertyOption(ControlText:="Abort on limit", ControlToolTip:="Abort twitter downloading when limit is reached", Category:=CAT_DOWN), PXML, PClonable>
        Friend Property AbortOnLimit As PropertyValue
        <PropertyOption(ControlText:="Download already parsed", ControlToolTip:="Download already parsed content on abort", Category:=CAT_DOWN), PXML, PClonable>
@@ -143,6 +145,7 @@ Namespace API.Twitter
        End Property
 #End Region
 #Region "Initializer"
+        Private Const SettingsVersionCurrent As Integer = 1
        Friend Sub New(ByVal AccName As String, ByVal Temp As Boolean)
            MyBase.New(TwitterSite, "x.com", AccName, Temp, My.Resources.SiteResources.TwitterIcon_32, My.Resources.SiteResources.TwitterIcon_32.ToBitmap)

@@ -153,7 +156,7 @@ Namespace API.Twitter
                .Cookies.Changed = False
            End With

-            UseNewIconXML = New PropertyValue(False)
+            UseNewIconXML = New PropertyValue(True)

            CookiesUpdate = New PropertyValue(False)
            UserAgentUse = New PropertyValue(True)
@@ -192,6 +195,10 @@ Namespace API.Twitter
            UseNetscapeCookies = True
        End Sub
        Friend Overrides Sub EndInit()
+            If Not SettingsVersion.Value = SettingsVersionCurrent Then
+                UseNewIconXML.Value = True
+                SettingsVersion.Value = SettingsVersionCurrent
+            End If
            UpdateIcon()
            MyBase.EndInit()
        End Sub
@@ -223,7 +230,7 @@ Namespace API.Twitter
        End Sub
        Friend Overrides Sub DownloadDone(ByVal What As ISiteSettings.Download)
            If UserNumber > 0 Then
-                If CBool(CookiesUpdate.Value) Then
+                If CBool(CookiesUpdate.Value) Or CookiesUpdateForce Then
                    With CookieKeeper.ParseNetscapeText(CookiesNetscapeFile.GetText(EDP.ReturnValue), EDP.ReturnValue)
                        If .ListExists Then
                            Responser.Cookies.Clear()
@@ -250,6 +257,7 @@ Namespace API.Twitter
                End With
            End If
            LIMIT_ABORT = False
+            CookiesUpdateForce = False
            MyBase.DownloadDone(What)
        End Sub
 #End Region
--- a/SCrawler/API/Twitter/UserData.vb
+++ b/SCrawler/API/Twitter/UserData.vb
@@ -30,6 +30,7 @@ Namespace API.Twitter
        Private Const Name_GifsSpecialFolder As String = "GifsSpecialFolder"
        Private Const Name_GifsPrefix As String = "GifsPrefix"
        Private Const Name_IsCommunity As String = "IsCommunity"
+        Private Const Name_LargeProfile As String = "LargeProfile"
        Private Const Name_DownloadModelChanged As String = "DownloadModelChanged"
 #End Region
 #Region "Declarations"
@@ -62,6 +63,47 @@ Namespace API.Twitter
        Friend Property GifsSpecialFolder As String = String.Empty
        Friend Property GifsPrefix As String = String.Empty
        Friend Property IsCommunity As Boolean = False
+#Region "LargeProfile"
+        Friend Property LargeProfile As Boolean = False
+        Private ReadOnly Property LargeProfileOverride As Boolean
+            Get
+                Return LargeProfile And Not FirstDownloadComplete
+            End Get
+        End Property
+        Private ReadOnly Property CookiesUpdate As Boolean
+            Get
+                If LargeProfileOverride Then
+                    MySettings.CookiesUpdateForce = True
+                    Return True
+                Else
+                    Return MySettings.CookiesUpdate.Value
+                End If
+            End Get
+        End Property
+        Private ReadOnly Property UserAgent As String
+            Get
+                If LargeProfileOverride Then
+                    Return MySettings.UserAgent(True).IfNullOrEmpty(Settings.UserAgent)
+                Else
+                    Return MySettings.UserAgent
+                End If
+            End Get
+        End Property
+        Private ReadOnly Property SleepTimerBeforeFirst As Integer
+            Get
+                Dim v% = MySettings.SleepTimerBeforeFirst.Value
+                If LargeProfileOverride And v <= 0 And v <> SiteSettings.TimerFirstUseTheSame Then v = SiteSettings.TimerFirstUseTheSame
+                Return v
+            End Get
+        End Property
+        Private ReadOnly Property SleepTimer As Integer
+            Get
+                Dim v% = MySettings.SleepTimer.Value
+                If LargeProfileOverride And v <= 0 Then v = SiteSettings.TimerDefault
+                Return v
+            End Get
+        End Property
+#End Region
        Private ReadOnly LikesPosts As List(Of String)
        Private ReadOnly PostsKV As List(Of PKV)
        Private ReadOnly _DataNames As List(Of String)
@@ -100,6 +142,7 @@ Namespace API.Twitter
                    DownloadModelForceApply = .DownloadModelForceApply
                    MediaModelAllowNonUserTweets = .MediaModelAllowNonUserTweets
                    DownloadBroadcasts = .DownloadBroadcasts
+                    LargeProfile = .LargeProfile
                    Dim dModel As DownloadModels = DownloadModel
                    If .DownloadModelMedia Then DownloadModel += DownloadModels.Media
                    If .DownloadModelProfile Or .DownloadBroadcasts Then DownloadModel += DownloadModels.Profile
@@ -155,6 +198,7 @@ Namespace API.Twitter
                    StartMD5Checked = .Value(Name_StartMD5Checked).FromXML(Of Boolean)(False)
                    MediaModelAllowNonUserTweets = .Value(Name_MediaModelAllowNonUserTweets).FromXML(Of Boolean)(False)
                    IsCommunity = .Value(Name_IsCommunity).FromXML(Of Boolean)(False)
+                    LargeProfile = .Value(Name_LargeProfile).FromXML(Of Boolean)(False)
                Else
                    If Name.Contains("@") And Not IsCommunity Then
                        IsCommunity = True
@@ -180,6 +224,7 @@ Namespace API.Twitter
                    .Add(Name_StartMD5Checked, StartMD5Checked.BoolToInteger)
                    .Add(Name_MediaModelAllowNonUserTweets, MediaModelAllowNonUserTweets.BoolToInteger)
                    .Add(Name_IsCommunity, IsCommunity.BoolToInteger)
+                    .Add(Name_LargeProfile, LargeProfile.BoolToInteger)
                    .Add(Name_TrueName, NameTrue(True))
                End If
            End With
@@ -615,6 +660,7 @@ nextpIndx:
                End If
                DownloadModelForceApply = False
                FirstDownloadComplete = True
+                LargeProfile = False
            Catch jsonNull_ex As JsonDocumentException When jsonNull_ex.State = WebDocumentEventArgs.States.Error
                Throw New Plugin.ExitException("No deserialized data found")
            Catch limit_ex As TwitterLimitException
@@ -839,8 +885,8 @@ nextpIndx:
        End Class
        Private ReadOnly Property SleepTimerValue(ByVal First As Boolean) As Integer
            Get
-                Dim fTimer% = If(First, MySettings.SleepTimerBeforeFirst, MySettings.SleepTimer).Value
-                If First And fTimer = SiteSettings.TimerFirstUseTheSame Then fTimer = MySettings.SleepTimer.Value
+                Dim fTimer% = If(First, SleepTimerBeforeFirst, SleepTimer)
+                If First And fTimer = SiteSettings.TimerFirstUseTheSame Then fTimer = SleepTimer
                Return fTimer
            End Get
        End Property
@@ -1060,10 +1106,10 @@ nextpIndx:
        Private Function GdlCreateConf(ByVal Path As SFile) As SFile
            Try
                Dim conf As SFile = $"{Path.PathWithSeparator}TwitterGdlConfig.conf"
-                Dim __userAgent$ = MySettings.UserAgent
+                Dim __userAgent$ = UserAgent
                If Not __userAgent.IsEmptyString Then __userAgent = $"""user-agent"": ""{__userAgent}"","
                Dim confText$ = "{""extractor"":{""cookies"": """ & MySettings.CookiesNetscapeFile.ToString.Replace("\", "/") &
-                                 $""",""cookies-update"": {IIf(CBool(MySettings.CookiesUpdate.Value), "true", "false")}," & __userAgent &
+                                 $""",""cookies-update"": {IIf(CookiesUpdate, "true", "false")}," & __userAgent &
                                 """twitter"":{""tweet-endpoint"": ""detail"",""cards"": false,""conversations"": true,""pinned"": false,""quoted"": false,""replies"": true,""retweets"": true,""strategy"": null,""text-tweets"": false,""twitpic"": false,""unique"": true,""users"": ""timeline"",""videos"": true}}}"
                If conf.Exists(SFO.Path, True, EDP.ThrowException) Then TextSaver.SaveTextToFile(confText, conf)
                If Not conf.Exists Then Throw New IO.FileNotFoundException("Can't find Twitter GDL config file", conf)
--- a/SCrawler/Editors/GlobalSettingsForm.vb
+++ b/SCrawler/Editors/GlobalSettingsForm.vb
@@ -187,24 +187,24 @@ Namespace Editors
        Private Sub MyDefs_ButtonOkClick(ByVal Sender As Object, ByVal e As KeyHandleEventArgs) Handles MyDefs.ButtonOkClick
            If MyDefs.MyFieldsChecker.AllParamsOK Then
                With Settings
-                    Dim a As Func(Of String, Object, Integer) =
-                        Function(t, v) MsgBoxE({$"You are set up higher than default count of along {t} downloading tasks." & vbNewLine &
-                                                $"Default: {SettingsCLS.DefaultMaxDownloadingTasks}" & vbNewLine &
-                                                $"Your value: {CInt(v)}" & vbNewLine &
-                                                "Increasing this value may lead to higher CPU usage." & vbNewLine &
-                                                "Do you really want to continue?",
-                                                "Increasing download tasks"},
-                                               vbExclamation,,, {"Confirm", $"Set to default ({SettingsCLS.DefaultMaxDownloadingTasks})", "Cancel"})
+                    Dim a As Func(Of String, Integer, Object, Integer) =
+                        Function(t, vc, v) MsgBoxE({$"You are set up higher than default count of along {t} downloading tasks." & vbNewLine &
+                                                    $"Default: {vc}" & vbNewLine &
+                                                    $"Your value: {CInt(v)}" & vbNewLine &
+                                                    "Increasing this value may lead to higher CPU usage." & vbNewLine &
+                                                    "Do you really want to continue?",
+                                                    "Increasing download tasks"},
+                                                   vbExclamation,,, {"Confirm", $"Set to default ({vc})", "Cancel"})

                    If CInt(TXT_MAX_JOBS_USERS.Value) > SettingsCLS.DefaultMaxDownloadingTasks Then
-                        Select Case a.Invoke("users", TXT_MAX_JOBS_USERS.Value)
+                        Select Case a.Invoke("users", SettingsCLS.DefaultMaxDownloadingTasks, TXT_MAX_JOBS_USERS.Value)
                            Case 1 : TXT_MAX_JOBS_USERS.Value = SettingsCLS.DefaultMaxDownloadingTasks
                            Case 2 : Exit Sub
                        End Select
                    End If
-                    If CInt(TXT_MAX_JOBS_CHANNELS.Value) > SettingsCLS.DefaultMaxDownloadingTasks Then
-                        Select Case a.Invoke("channels", TXT_MAX_JOBS_CHANNELS.Value)
-                            Case 1 : TXT_MAX_JOBS_CHANNELS.Value = SettingsCLS.DefaultMaxDownloadingTasks
+                    If CInt(TXT_MAX_JOBS_CHANNELS.Value) > SettingsCLS.DefaultMaxDownloadingTasks_Channels Then
+                        Select Case a.Invoke("channels", SettingsCLS.DefaultMaxDownloadingTasks_Channels, TXT_MAX_JOBS_CHANNELS.Value)
+                            Case 1 : TXT_MAX_JOBS_CHANNELS.Value = SettingsCLS.DefaultMaxDownloadingTasks_Channels
                            Case 2 : Exit Sub
                        End Select
                    End If
@@ -407,7 +407,7 @@ Namespace Editors
            If Sender.DefaultButton = ADB.Refresh Then TXT_MAX_JOBS_USERS.Value = SettingsCLS.DefaultMaxDownloadingTasks
        End Sub
        Private Sub TXT_MAX_JOBS_CHANNELS_ActionOnButtonClick(ByVal Sender As ActionButton, ByVal e As EventArgs) Handles TXT_MAX_JOBS_CHANNELS.ActionOnButtonClick
-            If Sender.DefaultButton = ADB.Refresh Then TXT_MAX_JOBS_CHANNELS.Value = SettingsCLS.DefaultMaxDownloadingTasks
+            If Sender.DefaultButton = ADB.Refresh Then TXT_MAX_JOBS_CHANNELS.Value = SettingsCLS.DefaultMaxDownloadingTasks_Channels
        End Sub
        Private Sub ChangePositionControlsEnabling() Handles OPT_FILE_NAME_REPLACE.CheckedChanged, OPT_FILE_NAME_ADD_DATE.CheckedChanged
            Dim b As Boolean = OPT_FILE_NAME_ADD_DATE.Checked And OPT_FILE_NAME_ADD_DATE.Enabled
--- a/Project/AssemblyInfo.vb
+++ b/Project/AssemblyInfo.vb
@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>

-<Assembly: AssemblyVersion("2025.7.18.0")>
-<Assembly: AssemblyFileVersion("2025.7.18.0")>
+<Assembly: AssemblyVersion("2025.8.1.0")>
+<Assembly: AssemblyFileVersion("2025.8.1.0")>
 <Assembly: NeutralResourcesLanguage("en")>
--- a/SCrawler/SettingsCLS.vb
+++ b/SCrawler/SettingsCLS.vb
@@ -23,6 +23,7 @@ Imports DoubleClickBehavior = SCrawler.DownloadObjects.STDownloader.DoubleClickB
 Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable
 #Region "Constants: defaults"
    Friend Const DefaultMaxDownloadingTasks As Integer = 5
+    Friend Const DefaultMaxDownloadingTasks_Channels As Integer = 1
    Friend Const TaskStackNamePornSite As String = "Porn sites"
    Friend Const Name_Node_Sites As String = "Sites"
    Private Const SitesValuesSeparator As String = ","
@@ -194,7 +195,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable
    Private ReadOnly BlackListFile As SFile = $"{SettingsFolderName}\BlackList.txt"
    Private ReadOnly UsersSettingsFile As SFile = $"{SettingsFolderName}\Users.xml"
    Private ReadOnly Property SettingsVersion As XMLValue(Of Integer)
-    Private Const SettingsVersionCurrent As Integer = 2
+    Private Const SettingsVersionCurrent As Integer = 3
    Friend ShortcutOpenFeed As New ButtonKey(Keys.F, True)
    Friend ShortcutOpenSearch As New ButtonKey(Keys.F,, True)
    Private Sub ChangeFeedOpenMode()
@@ -366,9 +367,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable
        ReparseMissingInTheRoutine = New XMLValue(Of Boolean)("ReparseMissingInTheRoutine", False, MyXML, n)
        UseDefaultAccountIfMissing = New XMLValue(Of Boolean)("UseDefaultAccountIfMissing", True, MyXML, n)
        AutomationBrushUndownloadedPlansMinutes = New XMLValue(Of Integer)("AutomationBrushUndownloadedPlansMinutes", 10080, MyXML, n)
-        DownDetectorEnabled = New XMLValue(Of Boolean)("DownDetectorEnabled", True, MyXML, n)
-        'TODELETE: DownDetectorEnabled change
-        If SettingsVersion.Value < SettingsVersionCurrent Then DownDetectorEnabled.Value = False 'SettingsVersionCurrent = 2
+        DownDetectorEnabled = New XMLValue(Of Boolean)("DownDetectorEnabled", False, MyXML, n)

        'Downloading: file naming
        n = {"Downloading", "FileName"}
@@ -392,7 +391,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable
        ChannelsDefaultReadyForDownload = New XMLValue(Of Boolean)("ChannelsDefaultReadyForDownload", False, MyXML, n)
        ChannelsDefaultTemporary = New XMLValue(Of Boolean)("ChannelsDefaultTemporary", True, MyXML, n)
        ChannelsHideExistsUser = New XMLValue(Of Boolean)("HideExistsUser", True, MyXML, n)
-        ChannelsMaxJobsCount = New XMLValue(Of Integer)("MaxJobsCount", DefaultMaxDownloadingTasks, MyXML, n)
+        ChannelsMaxJobsCount = New XMLValue(Of Integer)("MaxJobsCount", DefaultMaxDownloadingTasks_Channels, MyXML, n)
        n = {Name_Node_Sites, "Channels", "Users"}
        FromChannelDownloadTop = New XMLValue(Of Integer)("FromChannelDownloadTop", 10, MyXML, n)
        FromChannelDownloadTopUse = New XMLValue(Of Boolean)("FromChannelDownloadTopUse", False, MyXML, n)
@@ -497,6 +496,8 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable
        AdvancedFilter.IsViewFilter = True
        Labels.AddRange({AdvancedFilter}.GetGroupsLabels, False)

+        'TODELETE: DefaultMaxDownloadingTasks_Channels
+        If Not SettingsVersion = SettingsVersionCurrent Then ChannelsMaxJobsCount.Value = DefaultMaxDownloadingTasks_Channels 'SettingsVersionCurrent = 3
        SettingsVersion.Value = SettingsVersionCurrent

        MyXML.EndUpdate()