2025.8.1.0

YT
Update 'ReplaceModificationDate'

SCrawler
API.Instagram: fix 'LastCursor' issue
API.Reddit: add OAuth validation; add default credentials; hide unused controls; add 'SeparatedTasks'; bypass 429 error; fix crossposts downloading
API.Redgifs: force delete cookies if user added them
API.TikTok: yt-dlp modification (date change)
API.Twitter: simplify large profiles download
SettingsCLS: change default max value for channel downloads

Changelog.md

@@ -1,12 +1,26 @@
+# 2025.8.1.0
+
+*2025-08-01*
+
+- Added
+  - Sites:
+    - Reddit: **bypass error `429`**
+    - Twitter: **[large profile option](https://github.com/AAndyProgram/SCrawler/wiki/Settings#twitter-user-settings) in user settings**
+  - Minor improvements
+- Updated
+  - yt-dlp up to version **2025.27.21**
+  - gallery-dl up to version **1.30.2**
+- Fixed
+  - Reddit: in some cases crossposts don't download
+  - Minor bugs
+
 # 2025.7.18.0
 
 *2025-07-18*
 
 - Added
   - Sites:
-    - OnlyFans:
+    - OnlyFans: support for GIF files
-      - **bypass unpurchased videos**
-      - support for GIF files
     - Reddit: extended `429` error handling
     - Xhamster: support for downloading 'moments'
   - Minor improvements
@@ -14,6 +28,7 @@
   - yt-dlp up to version **2025.06.30**
   - gallery-dl up to version **1.30.0**
 - Fixed
+  - OnlyFans: **hanging on purchased content**
   - Minor bugs
 
 # 2025.6.12.0

SCrawler.YouTube/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2025.6.12.0")>
+<Assembly: AssemblyVersion("2025.8.1.0")>
-<Assembly: AssemblyFileVersion("2025.6.12.0")>
+<Assembly: AssemblyFileVersion("2025.8.1.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb

@@ -823,7 +823,9 @@ Namespace API.YouTube.Objects
                         'cmd = $"yt-dlp -f ""{cmd}"""
                         'cmd = $"yt-dlp -f {cmd}"
                         cmd = $"{YTDLP_NAME} -f {cmd}"
-                        If Not MyYouTubeSettings.ReplaceModificationDate Then cmd &= " --no-mtime"
+                        'yt-dlp 2025.07.21
+                        'If Not MyYouTubeSettings.ReplaceModificationDate Then cmd &= " --no-mtime"
+                        cmd &= $" --{IIf(MyYouTubeSettings.ReplaceModificationDate.Value, String.Empty, "no-")}mtime"
                         cmd.StringAppend(formats, " ")
                         cmd.StringAppend(subs, " ")
                         cmd.StringAppend(YouTubeFunctions.GetCookiesCommand(WithCookies, YouTubeCookieNetscapeFile), " ")

SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2025.6.12.0")>
+<Assembly: AssemblyVersion("2025.8.1.0")>
-<Assembly: AssemblyFileVersion("2025.6.12.0")>
+<Assembly: AssemblyFileVersion("2025.8.1.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler/API/Instagram/UserData.vb

@@ -412,6 +412,7 @@ Namespace API.Instagram
                 ThrowAny(Token)
                 HasError = False
                 Dim dt As Func(Of Boolean) = Function() (CBool(MySiteSettings.DownloadTimeline.Value) And GetTimeline) Or IsSavedPosts
+                If FirstLoadingDone Then LastCursor = String.Empty
                 If dt.Invoke And Not LastCursor.IsEmptyString Then
                     s = IIf(IsSavedPosts, Sections.SavedPosts, Sections.Timeline)
                     upClaimRequest.Invoke

SCrawler/API/Reddit/SiteSettings.vb

@@ -15,21 +15,32 @@ Imports PersonalUtilities.Tools.Web.Clients.Base
 Imports PersonalUtilities.Tools.Web.Documents.JSON
 Imports PersonalUtilities.Functions.XML
 Imports PersonalUtilities.Functions.RegularExpressions
+Imports DN = SCrawler.API.Base.DeclaredNames
 Imports DownDetector = SCrawler.API.Base.DownDetector
 Imports Download = SCrawler.Plugin.ISiteSettings.Download
 Namespace API.Reddit
-    <Manifest(RedditSiteKey), SavedPosts, SpecialForm(False), UseDownDetector>
+    <Manifest(RedditSiteKey), SavedPosts, SeparatedTasks, SpecialForm(False), UseDownDetector>
     Friend Class SiteSettings : Inherits SiteSettingsBase : Implements DownDetector.IDownDetector
 #Region "Declarations"
 #Region "Authorization"
+        Private Const ApiClientID_Default As String = "dYctRA-SIJxyykHe27lGZg"
+        Private Const ApiClientSecret_Default As String = "_5D6KzplRPDga-es1YlpzDIe9hiFlg"
         <PropertyOption(ControlText:="Login", ControlToolTip:="Your authorization username", IsAuth:=True), PXML, PClonable(Clone:=False)>
         Friend ReadOnly Property AuthUserName As PropertyValue
         <PropertyOption(ControlText:="Password", ControlToolTip:="Your authorization password", IsAuth:=True), PXML, PClonable(Clone:=False)>
         Friend ReadOnly Property AuthPassword As PropertyValue
         <PropertyOption(ControlText:="Client ID", ControlToolTip:="Your registered app client ID", IsAuth:=True), PXML, PClonable(Clone:=False)>
         Friend ReadOnly Property ApiClientID As PropertyValue
+        <PropertyUpdater(NameOf(ApiClientID))> Private Function ApiClientID_SetDefault() As Boolean
+            ApiClientID.Value = ApiClientID_Default
+            Return True
+        End Function
         <PropertyOption(ControlText:="Client Secret", ControlToolTip:="Your registered app client secret", IsAuth:=True), PXML, PClonable(Clone:=False)>
         Friend ReadOnly Property ApiClientSecret As PropertyValue
+        <PropertyUpdater(NameOf(ApiClientSecret))> Private Function ApiClientSecret_SetDefault() As Boolean
+            ApiClientSecret.Value = ApiClientSecret_Default
+            Return True
+        End Function
         <PropertyOption(ControlText:="Bearer token",
                         ControlToolTip:="Bearer token (can be null)." & vbCr &
                                         "If you are using cookies to download the timeline, it is highly recommended that you add a token." & vbCr &
@@ -103,12 +114,15 @@ Namespace API.Reddit
         End Function
 #End Region
 #Region "Other"
-        <PropertyOption(ControlText:="Use M3U8", ControlToolTip:="Use M3U8 or mp4 for Reddit videos", IsAuth:=False), PXML, PClonable>
+        <PropertyOption(ControlText:="Use M3U8", ControlToolTip:="Use M3U8 or mp4 for Reddit videos"), PXML, PClonable, HiddenControl>
         Friend ReadOnly Property UseM3U8 As PropertyValue
-        <PropertyOption(ControlText:="Check image", ControlToolTip:="Check the image if it exists before downloading (it makes downloading very slow)", IsAuth:=False), PXML, PClonable>
+        <PropertyOption(ControlText:="Check image", ControlToolTip:="Check the image if it exists before downloading (it makes downloading very slow)"), PXML, PClonable, HiddenControl>
         Friend ReadOnly Property CheckImage As PropertyValue
-        <PropertyOption(ControlText:="Check image: get original", ControlToolTip:="Get the original image if it exists", IsAuth:=False), PXML, PClonable>
+        <PropertyOption(ControlText:="Check image: get original", ControlToolTip:="Get the original image if it exists"), PXML, PClonable, HiddenControl>
         Friend ReadOnly Property CheckImageReturnOrig As PropertyValue
+        <PropertyOption(ControlText:=DN.ConcurrentDownloadsCaption,
+                        ControlToolTip:=DN.ConcurrentDownloadsToolTip, AllowNull:=False), PXML, TaskCounter, PClonable>
+        Friend ReadOnly Property ConcurrentDownloads As PropertyValue
 #End Region
 #Region "IDownDetector Support"
         Private ReadOnly Property IDownDetector_Value As Integer Implements DownDetector.IDownDetector.Value
@@ -160,6 +174,7 @@ Namespace API.Reddit
             UseM3U8 = New PropertyValue(True)
             CheckImage = New PropertyValue(False)
             CheckImageReturnOrig = New PropertyValue(True)
+            ConcurrentDownloads = New PropertyValue(1)
 
             MDD = New MyDownDetector(Me)
 
@@ -167,10 +182,13 @@ Namespace API.Reddit
             ImageVideoContains = "reddit.com"
             UserRegex = RParams.DM("[htps:/]{7,8}.*?reddit.com/([user]{1,4})/([^/\?&]+)", 0, RegexReturn.ListByMatch, EDP.ReturnValue)
         End Sub
-        Private Const SettingsVersionCurrent As Integer = 2
+        Private Const SettingsVersionCurrent As Integer = 3
         Friend Overrides Sub EndInit()
             If CInt(SettingsVersion.Value) < SettingsVersionCurrent Then
                 SettingsVersion.Value = SettingsVersionCurrent
+                UseM3U8.Value = True
+                CheckImage.Value = False
+                CheckImageReturnOrig.Value = True
                 BearerTokenUseCurl.Value = False
             End If
             MyBase.EndInit()
@@ -208,6 +226,7 @@ Namespace API.Reddit
             End Sub
         End Class
         Friend Property SessionInterrupted As Boolean = False
+        Friend Property RequestCount As Integer = 0
         Friend Overrides Function ReadyToDownload(ByVal What As Download) As Boolean
             If What = Download.Main Then
                 Return Not SessionInterrupted
@@ -223,6 +242,7 @@ Namespace API.Reddit
         End Function
         Friend Overrides Sub DownloadDone(ByVal What As Download)
             SessionInterrupted = False
+            RequestCount = 0
             MDD.Reset()
             MyBase.DownloadDone(What)
         End Sub

SCrawler/API/Reddit/UserData.vb

@@ -8,19 +8,20 @@
 ' but WITHOUT ANY WARRANTY
 Imports System.Net
 Imports System.Threading
+Imports PersonalUtilities.Functions.RegularExpressions
+Imports PersonalUtilities.Functions.XML
+Imports PersonalUtilities.Tools.ImageRenderer
+Imports PersonalUtilities.Tools.Web.Clients
+Imports PersonalUtilities.Tools.Web.Clients.Base
+Imports PersonalUtilities.Tools.Web.Documents.JSON
 Imports SCrawler.API.Base
 Imports SCrawler.API.Reddit.RedditViewExchange
 Imports SCrawler.API.YouTube.Objects
 Imports SCrawler.Plugin.Hosts
-Imports PersonalUtilities.Functions.XML
+Imports CPeriod = SCrawler.API.Reddit.IRedditView.Period
-Imports PersonalUtilities.Functions.RegularExpressions
+Imports CView = SCrawler.API.Reddit.IRedditView.View
-Imports PersonalUtilities.Tools.ImageRenderer
-Imports PersonalUtilities.Tools.Web.Clients
-Imports PersonalUtilities.Tools.Web.Documents.JSON
 Imports UStates = SCrawler.API.Base.UserMedia.States
 Imports UTypes = SCrawler.API.Base.UserMedia.Types
-Imports CView = SCrawler.API.Reddit.IRedditView.View
-Imports CPeriod = SCrawler.API.Reddit.IRedditView.Period
 Namespace API.Reddit
     Friend Class UserData : Inherits UserDataBase : Implements IChannelLimits, IRedditView
 #Region "Declarations"
@@ -269,6 +270,8 @@ Namespace API.Reddit
                 End If
             End With
 
+            Responser.ProcessExceptionDecision = AddressOf Err429Process
+
             _TotalPostsDownloaded = 0
             If IsSavedPosts Then
                 Responser.DecodersError = EDP.ReturnValue
@@ -304,6 +307,7 @@ Namespace API.Reddit
 #End Region
 #Region "Download Functions (User, Channel)"
         Private Err429Count As Integer = 0
+        Private Err429TryAgain As Boolean = False
         Private _TotalPostsDownloaded As Integer = 0
         Private ReadOnly _CrossPosts As List(Of String)
         Private Const SiteGfycatKey As String = "gfycat"
@@ -311,6 +315,28 @@ Namespace API.Reddit
         Private Const Node_CrosspostRootId As String = "crosspostRootId"
         Private Const Node_CrosspostParentId As String = "crosspostParentId"
         Private Const Node_CrosspostParent As String = "crosspost_parent"
+        Private Sub Wait429()
+            With MySiteSettings
+                If Not Err429TryAgain Then .RequestCount += 1
+                Err429TryAgain = False
+                If (.RequestCount Mod 100) = 0 Then Thread.Sleep(60100)
+            End With
+        End Sub
+        Private Function Err429Process(ByVal Status As IResponserStatus, ByVal NullArg As Object, ByVal CurrErr As ErrorsDescriber) As ErrorsDescriber
+            If Not Status Is Nothing AndAlso Status.StatusCode = 429 Then
+                If Err429Count = 0 Then
+                    Err429Count += 1
+                    MySiteSettings.RequestCount = 100
+                    Err429TryAgain = True
+                    Return EDP.ReturnValue
+                End If
+            End If
+            Return CurrErr
+        End Function
+        Private Sub Err429Reset()
+            Err429Count = 0
+            Err429TryAgain = False
+        End Sub
         Private Sub DownloadDataUser(ByVal POST As String, ByVal Token As CancellationToken)
             Dim eObj% = 0
             Dim round% = 0
@@ -331,8 +357,10 @@ Namespace API.Reddit
                     'URL = $"https://gateway.reddit.com/desktopapi/v1/user/{NameTrue}/posts?rtj=only&allow_quarantined=true&allow_over18=1&include=identity&after={POST}&dist=25&sort={View}&t={Period}&layout=classic"
                     URL = $"https://oauth.reddit.com/user/{NameTrue}/submitted.json?rtj=only&allow_quarantined=true&allow_over18=1&include=identity&after={POST}&dist=25&sort={View}&t={Period}&layout=classic"
                     ThrowAny(Token)
+                    Wait429()
                     Dim r$ = Responser.GetResponse(URL)
                     If Not r.IsEmptyString Then
+                        Err429Reset()
                         Using w As EContainer = JsonDocument.Parse(r).XmlIfNothing
                             If w.Count > 0 Then
                                 'n = w.GetNode(JsonNodesJson)
@@ -347,6 +375,7 @@ Namespace API.Reddit
                                                 If CheckNode(.Self) Then
 
                                                     'Obtain post ID
+                                                    PostID = String.Empty
                                                     PostTmp = .Value("name") '.Name
                                                     If PostTmp.IsEmptyString Then PostTmp = .Value("id")
                                                     If PostTmp.IsEmptyString Then Continue For
@@ -354,8 +383,9 @@ Namespace API.Reddit
                                                     If IsCrossPost(.Self) Then
                                                         _CrossPosts.ListAddList({ .Value(Node_CrosspostRootId),
                                                                                   .Value(Node_CrosspostParentId),
-                                                                                  .Value(Node_CrosspostParent)}, LNC)
+                                                                                  .Value(Node_CrosspostParent),
-                                                        Continue For
+                                                                                  PostTmp}, LNC)
+                                                        If ParseUserMediaOnly Then Continue For
                                                     Else
                                                         If Not _CrossPosts.Contains(PostTmp) Then PostID = PostTmp : PostTmp = String.Empty
                                                     End If
@@ -384,6 +414,8 @@ Namespace API.Reddit
                         End Using
                         If POST.IsEmptyString And ExistsDetected Then Exit Sub
                         If Not _PostID().IsEmptyString And NewPostDetected Then DownloadDataUser(_PostID(), Token)
+                    ElseIf Err429TryAgain Then
+                        Continue Do
                     End If
                     _completed = True
                 Catch ex As Exception
@@ -420,9 +452,11 @@ Namespace API.Reddit
                     End If
 
                     ThrowAny(Token)
+                    Wait429()
                     Dim r$ = Responser.GetResponse(URL)
-                    If IsSavedPosts Then Err429Count = 0
+                    'If IsSavedPosts Then Err429Count = 0
                     If Not r.IsEmptyString Then
+                        Err429Reset()
                         Using w As EContainer = JsonDocument.Parse(r).XmlIfNothing
                             If w.Count > 0 Then
                                 n = w.GetNode(ChannelJsonNodes)
@@ -479,6 +513,8 @@ Namespace API.Reddit
                         End Using
                         If POST.IsEmptyString And ExistsDetected Then Exit Sub
                         If Not PostID.IsEmptyString And NewPostDetected Then DownloadDataChannel(PostID, Token)
+                    ElseIf Err429TryAgain Then
+                        Continue Do
                     End If
                     _completed = True
                 Catch ex As Exception
@@ -496,11 +532,13 @@ Namespace API.Reddit
         End Sub
 #End Region
 #Region "GetUserInfo"
-        Private Sub GetUserInfo()
+        Private Sub GetUserInfo(Optional ByVal Round As Integer = 0)
             Try
                 If Not IsSavedPosts And ChannelInfo Is Nothing Then
+                    Wait429()
                     Dim r$ = Responser.GetResponse($"https://reddit.com/{IIf(IsChannel, "r", "user")}/{NameTrue}/about.json",, EDP.ReturnValue)
                     If Not r.IsEmptyString Then
+                        Err429Reset()
                         Using j As EContainer = JsonDocument.Parse(r)
                             If Not j Is Nothing AndAlso j.Contains({"data", "subreddit"}) Then
                                 If ID.IsEmptyString Then ID = j.Value({"data"}, "id")
@@ -516,6 +554,8 @@ Namespace API.Reddit
                                 End With
                             End If
                         End Using
+                    ElseIf Err429TryAgain And Round < 2 Then
+                        GetUserInfo(Round + 1)
                     End If
                 End If
             Catch ex As Exception
@@ -631,8 +671,11 @@ Namespace API.Reddit
                         Else
                             Dim tPostId$ = e.Value(Node_CrosspostParent).IfNullOrEmpty(e.Value(Node_CrosspostParentId)).IfNullOrEmpty(e.Value(Node_CrosspostRootId))
                             If Not PostID.IsEmptyString Then
+                                For ri% = 0 To 1
+                                    Wait429()
                                     Dim r$ = Responser.GetResponse($"https://www.reddit.com/comments/{tPostId.Split("_").LastOrDefault}/.json",, EDP.ReturnValue)
                                     If Not r.IsEmptyString Then
+                                        Err429Reset()
                                         Using j As EContainer = JsonDocument.Parse(r, EDP.ReturnValue)
                                             If j.ListExists Then
                                                 With j.ItemF({0, "data", "children", 0, "data"})
@@ -640,7 +683,9 @@ Namespace API.Reddit
                                                 End With
                                             End If
                                         End Using
+                                        Exit For
                                     End If
+                                Next
                             End If
                         End If
                     End If
@@ -906,7 +951,10 @@ Namespace API.Reddit
                                         End If
                                         Continue For
                                     Else
+                                        Wait429()
                                         r = Responser.GetResponse(m.URL,, e)
+                                        If r.IsEmptyString And Err429TryAgain Then _repeatForRedgifs = True
+                                        If Not r.IsEmptyString Then Err429Reset()
                                     End If
                                 Loop While _repeatForRedgifs
                             Else
@@ -944,11 +992,13 @@ Namespace API.Reddit
                     RedGifsResponser = RedGifsHost.Responser.Copy
                     Dim respNoHeaders As Responser = Responser.Copy
                     Dim m As UserMedia, m2 As UserMedia
-                    Dim r$, url$
+                    Dim r$ = String.Empty, url$
+                    Dim ri As Byte
                     Dim j As EContainer
                     Dim lastCount%, li%
                     Dim rv As New ErrorsDescriber(EDP.ReturnValue)
                     respNoHeaders.Headers.Clear()
+                    respNoHeaders.ProcessExceptionDecision = AddressOf Err429Process
                     ProgressPre.ChangeMax(_ContentList.Count)
                     For i% = 0 To _ContentList.Count - 1
                         m = _ContentList(i)
@@ -956,9 +1006,14 @@ Namespace API.Reddit
                         If m.State = UStates.Missing AndAlso Not m.Post.ID.IsEmptyString Then
                             ThrowAny(Token)
                             url = $"https://www.reddit.com/comments/{m.Post.ID.Split("_").LastOrDefault}/.json"
+                            For ri = 0 To 1
+                                Wait429()
                                 r = Responser.GetResponse(url,, rv)
-                            If r.IsEmptyString Then r = respNoHeaders.GetResponse(url,, rv)
+                                If r.IsEmptyString Then Wait429() : r = respNoHeaders.GetResponse(url,, rv)
+                                If Not (r.IsEmptyString And Err429TryAgain) Then Exit For
+                            Next
                             If Not r.IsEmptyString Then
+                                Err429Reset()
                                 j = JsonDocument.Parse(r, rv)
                                 If Not j Is Nothing Then
                                     If j.Count > 0 Then
@@ -1101,13 +1156,22 @@ Namespace API.Reddit
                 ElseIf .StatusCode = HttpStatusCode.InternalServerError Then '500
                     If Not IsNothing(EObj) AndAlso IsNumeric(EObj) AndAlso CInt(EObj) = HttpStatusCode.InternalServerError Then Return 1
                     Return HttpStatusCode.InternalServerError
-                ElseIf .StatusCode = 429 And IsSavedPosts And Err429Count = 0 Then '429 (saved)
+                    'ElseIf .StatusCode = 429 And IsSavedPosts And Err429Count = 0 Then '429 (saved)
-                    Err429Count += 1
+                    '    Err429Count += 1
-                    Return 429
+                    '    Return 429
                 ElseIf .StatusCode = 429 Then '429 (all)
+                    'If ((Not IsSavedPosts And CBool(MySiteSettings.UseTokenForTimelines.Value)) Or (IsSavedPosts And CBool(MySiteSettings.UseTokenForSavedPosts.Value))) AndAlso
+                    '   Not MySiteSettings.CredentialsExists Then
+                    '    LogError(Nothing, "[429] You should use OAuth authorization or disable " &
+                    '                      IIf(IsSavedPosts, "token usage for downloading saved posts", "the use of token and cookies for downloading timelines"))
+                    'Else
+                    '    LogError(Nothing, "Too many requests (429). Try again later!")
+                    'End If
+                    'MySiteSettings.SessionInterrupted = True
+                    'Throw New Plugin.ExitException With {.Silent = True}
                     If ((Not IsSavedPosts And CBool(MySiteSettings.UseTokenForTimelines.Value)) Or (IsSavedPosts And CBool(MySiteSettings.UseTokenForSavedPosts.Value))) AndAlso
                        Not MySiteSettings.CredentialsExists Then
-                        LogError(Nothing, $"[{CInt(Responser.StatusCode)}] You should use OAuth authorization or disable " &
+                        LogError(Nothing, "[429] You should use OAuth authorization or disable " &
                                           IIf(IsSavedPosts, "token usage for downloading saved posts", "the use of token and cookies for downloading timelines"))
                     Else
                         LogError(Nothing, "Too many requests (429). Try again later!")

SCrawler/API/Redgifs/SiteSettings.vb

@@ -22,6 +22,7 @@ Namespace API.RedGifs
         Friend ReadOnly Property Token As PropertyValue
         <PropertyOption, ControlNumber(2), PClonable, HiddenControl>
         Private ReadOnly Property UserAgent As PropertyValue
+        <PXML> Friend ReadOnly Property UseCookies As PropertyValue
         <PXML> Friend ReadOnly Property TokenLastDateUpdated As PropertyValue
         Private Const TokenName As String = "authorization"
 #Region "TokenUpdateInterval"
@@ -47,6 +48,7 @@ Namespace API.RedGifs
             End With
             Token = New PropertyValue(t, GetType(String), Sub(v) UpdateResponse(NameOf(Token), v))
             UserAgent = New PropertyValue(If(Responser.UserAgentExists, Responser.UserAgent, String.Empty), GetType(String), Sub(v) UpdateResponse(NameOf(UserAgent), v))
+            UseCookies = New PropertyValue(False)
             TokenLastDateUpdated = New PropertyValue(Now.AddYears(-1), GetType(Date))
             TokenUpdateInterval = New PropertyValue(60 * 12, GetType(Integer))
             TokenUpdateIntervalProvider = New TokenRefreshIntervalProvider

SCrawler/API/Redgifs/UserData.vb

@@ -36,6 +36,7 @@ Namespace API.RedGifs
 #End Region
 #Region "Download functions"
         Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
+            If Not MySettings.UseCookies.Value Then Responser.Cookies.Clear()
             DownloadData(1, Token)
         End Sub
         Private Overloads Sub DownloadData(ByVal Page As Integer, ByVal Token As CancellationToken)

SCrawler/API/TikTok/UserData.vb

@@ -422,7 +422,11 @@ Namespace API.TikTok
             End If
             If DateBefore.HasValue Then command &= $"--datebefore {DateBefore.Value.AddDays(1).ToStringDate(SimpleDateConverter)} "
             If DateAfter.HasValue Then command &= $"--dateafter {DateAfter.Value.AddDays(-1).ToStringDate(SimpleDateConverter)} "
-            If Not CBool(If(IsSingleObjectDownload, MySettings.UseParsedVideoDateSTD, MySettings.UseParsedVideoDate).Value) Then command &= "--no-mtime "
+            If Not CBool(If(IsSingleObjectDownload, MySettings.UseParsedVideoDateSTD, MySettings.UseParsedVideoDate).Value) Then
+                command &= "--no-mtime "
+            Else
+                command &= "--mtime "
+            End If
             If MySettings.CookiesNetscapeFile.Exists Then command &= $"--no-cookies-from-browser --cookies ""{MySettings.CookiesNetscapeFile}"" "
             command &= $"{URL} "
             If SupportOutput Then

SCrawler/API/Twitter/EditorExchangeOptions.vb

@@ -50,6 +50,10 @@ Namespace API.Twitter
                   Caption:="Force apply",
                   ToolTip:="Force overrides the default parameters for the first download." & vbCr & "Applies to first download only.", LeftOffset:=DefaultOffset)>
         Friend Overridable Property DownloadModelForceApply As Boolean = False
+        <PSetting(Address:=SettingAddress.User,
+                  Caption:="Large profile",
+                  ToolTip:="This setting is only used on the first download and is intended to temporarily override the default site settings if they are incompatible with downloading large profiles. After the first download is complete, this option will be disabled and cannot be enabled again.")>
+        Friend Overridable Property LargeProfile As Boolean = False
         Private ReadOnly Property MySettings As Object
         Friend Sub New(ByVal s As SiteSettings)
             MyBase.New(s)
@@ -76,6 +80,7 @@ Namespace API.Twitter
             UseMD5Comparison = u.UseMD5Comparison
             RemoveExistingDuplicates = u.RemoveExistingDuplicates
             MediaModelAllowNonUserTweets = u.MediaModelAllowNonUserTweets
+            LargeProfile = u.LargeProfile
             If Not TypeOf u Is Mastodon.UserData Then
                 DownloadModelForceApply = u.DownloadModelForceApply
                 DownloadBroadcasts = u.DownloadBroadcasts

SCrawler/API/Twitter/SiteSettings.vb

@@ -38,6 +38,7 @@ Namespace API.Twitter
         Private Const CAT_DOWN As String = "Downloading"
 #End Region
 #Region "Auth"
+        Friend Property CookiesUpdateForce As Boolean = False
         <PropertyOption(ControlText:="Update cookies", ControlToolTip:="Update cookies during requests", IsAuth:=True), PXML, PClonable, HiddenControl>
         Friend ReadOnly Property CookiesUpdate As PropertyValue
         <PropertyOption(ControlText:="Use UserAgent", ControlToolTip:="Use UserAgent in requests", IsAuth:=True), PXML, PClonable>
@@ -45,9 +46,9 @@ Namespace API.Twitter
         <PropertyOption(ControlText:="UserAgent", IsAuth:=True, AllowNull:=True, InheritanceName:=SettingsCLS.HEADER_DEF_UserAgent),
          PXML("UserAgent", OnlyForChecked:=True), PClonable>
         Private ReadOnly Property UserAgentXML As PropertyValue
-        Friend ReadOnly Property UserAgent As String
+        Friend ReadOnly Property UserAgent(Optional ByVal Force As Boolean = False) As String
             Get
-                If CBool(UserAgentUse.Value) AndAlso Not CStr(UserAgentXML.Value).IsEmptyString Then
+                If (CBool(UserAgentUse.Value) Or Force) AndAlso Not CStr(UserAgentXML.Value).IsEmptyString Then
                     Return UserAgentXML.Value
                 Else
                     Return String.Empty
@@ -73,6 +74,7 @@ Namespace API.Twitter
 #Region "Limits"
         Friend Const TimerDisabled As Integer = -1
         Friend Const TimerFirstUseTheSame As Integer = -2
+        Friend Const TimerDefault As Integer = 20
         <PropertyOption(ControlText:="Abort on limit", ControlToolTip:="Abort twitter downloading when limit is reached", Category:=CAT_DOWN), PXML, PClonable>
         Friend Property AbortOnLimit As PropertyValue
         <PropertyOption(ControlText:="Download already parsed", ControlToolTip:="Download already parsed content on abort", Category:=CAT_DOWN), PXML, PClonable>
@@ -143,6 +145,7 @@ Namespace API.Twitter
         End Property
 #End Region
 #Region "Initializer"
+        Private Const SettingsVersionCurrent As Integer = 1
         Friend Sub New(ByVal AccName As String, ByVal Temp As Boolean)
             MyBase.New(TwitterSite, "x.com", AccName, Temp, My.Resources.SiteResources.TwitterIcon_32, My.Resources.SiteResources.TwitterIcon_32.ToBitmap)
 
@@ -153,7 +156,7 @@ Namespace API.Twitter
                 .Cookies.Changed = False
             End With
 
-            UseNewIconXML = New PropertyValue(False)
+            UseNewIconXML = New PropertyValue(True)
 
             CookiesUpdate = New PropertyValue(False)
             UserAgentUse = New PropertyValue(True)
@@ -192,6 +195,10 @@ Namespace API.Twitter
             UseNetscapeCookies = True
         End Sub
         Friend Overrides Sub EndInit()
+            If Not SettingsVersion.Value = SettingsVersionCurrent Then
+                UseNewIconXML.Value = True
+                SettingsVersion.Value = SettingsVersionCurrent
+            End If
             UpdateIcon()
             MyBase.EndInit()
         End Sub
@@ -223,7 +230,7 @@ Namespace API.Twitter
         End Sub
         Friend Overrides Sub DownloadDone(ByVal What As ISiteSettings.Download)
             If UserNumber > 0 Then
-                If CBool(CookiesUpdate.Value) Then
+                If CBool(CookiesUpdate.Value) Or CookiesUpdateForce Then
                     With CookieKeeper.ParseNetscapeText(CookiesNetscapeFile.GetText(EDP.ReturnValue), EDP.ReturnValue)
                         If .ListExists Then
                             Responser.Cookies.Clear()
@@ -250,6 +257,7 @@ Namespace API.Twitter
                 End With
             End If
             LIMIT_ABORT = False
+            CookiesUpdateForce = False
             MyBase.DownloadDone(What)
         End Sub
 #End Region

SCrawler/API/Twitter/UserData.vb

@@ -30,6 +30,7 @@ Namespace API.Twitter
         Private Const Name_GifsSpecialFolder As String = "GifsSpecialFolder"
         Private Const Name_GifsPrefix As String = "GifsPrefix"
         Private Const Name_IsCommunity As String = "IsCommunity"
+        Private Const Name_LargeProfile As String = "LargeProfile"
         Private Const Name_DownloadModelChanged As String = "DownloadModelChanged"
 #End Region
 #Region "Declarations"
@@ -62,6 +63,47 @@ Namespace API.Twitter
         Friend Property GifsSpecialFolder As String = String.Empty
         Friend Property GifsPrefix As String = String.Empty
         Friend Property IsCommunity As Boolean = False
+#Region "LargeProfile"
+        Friend Property LargeProfile As Boolean = False
+        Private ReadOnly Property LargeProfileOverride As Boolean
+            Get
+                Return LargeProfile And Not FirstDownloadComplete
+            End Get
+        End Property
+        Private ReadOnly Property CookiesUpdate As Boolean
+            Get
+                If LargeProfileOverride Then
+                    MySettings.CookiesUpdateForce = True
+                    Return True
+                Else
+                    Return MySettings.CookiesUpdate.Value
+                End If
+            End Get
+        End Property
+        Private ReadOnly Property UserAgent As String
+            Get
+                If LargeProfileOverride Then
+                    Return MySettings.UserAgent(True).IfNullOrEmpty(Settings.UserAgent)
+                Else
+                    Return MySettings.UserAgent
+                End If
+            End Get
+        End Property
+        Private ReadOnly Property SleepTimerBeforeFirst As Integer
+            Get
+                Dim v% = MySettings.SleepTimerBeforeFirst.Value
+                If LargeProfileOverride And v <= 0 And v <> SiteSettings.TimerFirstUseTheSame Then v = SiteSettings.TimerFirstUseTheSame
+                Return v
+            End Get
+        End Property
+        Private ReadOnly Property SleepTimer As Integer
+            Get
+                Dim v% = MySettings.SleepTimer.Value
+                If LargeProfileOverride And v <= 0 Then v = SiteSettings.TimerDefault
+                Return v
+            End Get
+        End Property
+#End Region
         Private ReadOnly LikesPosts As List(Of String)
         Private ReadOnly PostsKV As List(Of PKV)
         Private ReadOnly _DataNames As List(Of String)
@@ -100,6 +142,7 @@ Namespace API.Twitter
                     DownloadModelForceApply = .DownloadModelForceApply
                     MediaModelAllowNonUserTweets = .MediaModelAllowNonUserTweets
                     DownloadBroadcasts = .DownloadBroadcasts
+                    LargeProfile = .LargeProfile
                     Dim dModel As DownloadModels = DownloadModel
                     If .DownloadModelMedia Then DownloadModel += DownloadModels.Media
                     If .DownloadModelProfile Or .DownloadBroadcasts Then DownloadModel += DownloadModels.Profile
@@ -155,6 +198,7 @@ Namespace API.Twitter
                     StartMD5Checked = .Value(Name_StartMD5Checked).FromXML(Of Boolean)(False)
                     MediaModelAllowNonUserTweets = .Value(Name_MediaModelAllowNonUserTweets).FromXML(Of Boolean)(False)
                     IsCommunity = .Value(Name_IsCommunity).FromXML(Of Boolean)(False)
+                    LargeProfile = .Value(Name_LargeProfile).FromXML(Of Boolean)(False)
                 Else
                     If Name.Contains("@") And Not IsCommunity Then
                         IsCommunity = True
@@ -180,6 +224,7 @@ Namespace API.Twitter
                     .Add(Name_StartMD5Checked, StartMD5Checked.BoolToInteger)
                     .Add(Name_MediaModelAllowNonUserTweets, MediaModelAllowNonUserTweets.BoolToInteger)
                     .Add(Name_IsCommunity, IsCommunity.BoolToInteger)
+                    .Add(Name_LargeProfile, LargeProfile.BoolToInteger)
                     .Add(Name_TrueName, NameTrue(True))
                 End If
             End With
@@ -615,6 +660,7 @@ nextpIndx:
                 End If
                 DownloadModelForceApply = False
                 FirstDownloadComplete = True
+                LargeProfile = False
             Catch jsonNull_ex As JsonDocumentException When jsonNull_ex.State = WebDocumentEventArgs.States.Error
                 Throw New Plugin.ExitException("No deserialized data found")
             Catch limit_ex As TwitterLimitException
@@ -839,8 +885,8 @@ nextpIndx:
         End Class
         Private ReadOnly Property SleepTimerValue(ByVal First As Boolean) As Integer
             Get
-                Dim fTimer% = If(First, MySettings.SleepTimerBeforeFirst, MySettings.SleepTimer).Value
+                Dim fTimer% = If(First, SleepTimerBeforeFirst, SleepTimer)
-                If First And fTimer = SiteSettings.TimerFirstUseTheSame Then fTimer = MySettings.SleepTimer.Value
+                If First And fTimer = SiteSettings.TimerFirstUseTheSame Then fTimer = SleepTimer
                 Return fTimer
             End Get
         End Property
@@ -1060,10 +1106,10 @@ nextpIndx:
         Private Function GdlCreateConf(ByVal Path As SFile) As SFile
             Try
                 Dim conf As SFile = $"{Path.PathWithSeparator}TwitterGdlConfig.conf"
-                Dim __userAgent$ = MySettings.UserAgent
+                Dim __userAgent$ = UserAgent
                 If Not __userAgent.IsEmptyString Then __userAgent = $"""user-agent"": ""{__userAgent}"","
                 Dim confText$ = "{""extractor"":{""cookies"": """ & MySettings.CookiesNetscapeFile.ToString.Replace("\", "/") &
-                                 $""",""cookies-update"": {IIf(CBool(MySettings.CookiesUpdate.Value), "true", "false")}," & __userAgent &
+                                 $""",""cookies-update"": {IIf(CookiesUpdate, "true", "false")}," & __userAgent &
                                  """twitter"":{""tweet-endpoint"": ""detail"",""cards"": false,""conversations"": true,""pinned"": false,""quoted"": false,""replies"": true,""retweets"": true,""strategy"": null,""text-tweets"": false,""twitpic"": false,""unique"": true,""users"": ""timeline"",""videos"": true}}}"
                 If conf.Exists(SFO.Path, True, EDP.ThrowException) Then TextSaver.SaveTextToFile(confText, conf)
                 If Not conf.Exists Then Throw New IO.FileNotFoundException("Can't find Twitter GDL config file", conf)

SCrawler/Editors/GlobalSettingsForm.vb

@@ -187,24 +187,24 @@ Namespace Editors
         Private Sub MyDefs_ButtonOkClick(ByVal Sender As Object, ByVal e As KeyHandleEventArgs) Handles MyDefs.ButtonOkClick
             If MyDefs.MyFieldsChecker.AllParamsOK Then
                 With Settings
-                    Dim a As Func(Of String, Object, Integer) =
+                    Dim a As Func(Of String, Integer, Object, Integer) =
-                        Function(t, v) MsgBoxE({$"You are set up higher than default count of along {t} downloading tasks." & vbNewLine &
+                        Function(t, vc, v) MsgBoxE({$"You are set up higher than default count of along {t} downloading tasks." & vbNewLine &
-                                                $"Default: {SettingsCLS.DefaultMaxDownloadingTasks}" & vbNewLine &
+                                                    $"Default: {vc}" & vbNewLine &
                                                     $"Your value: {CInt(v)}" & vbNewLine &
                                                     "Increasing this value may lead to higher CPU usage." & vbNewLine &
                                                     "Do you really want to continue?",
                                                     "Increasing download tasks"},
-                                               vbExclamation,,, {"Confirm", $"Set to default ({SettingsCLS.DefaultMaxDownloadingTasks})", "Cancel"})
+                                                   vbExclamation,,, {"Confirm", $"Set to default ({vc})", "Cancel"})
 
                     If CInt(TXT_MAX_JOBS_USERS.Value) > SettingsCLS.DefaultMaxDownloadingTasks Then
-                        Select Case a.Invoke("users", TXT_MAX_JOBS_USERS.Value)
+                        Select Case a.Invoke("users", SettingsCLS.DefaultMaxDownloadingTasks, TXT_MAX_JOBS_USERS.Value)
                             Case 1 : TXT_MAX_JOBS_USERS.Value = SettingsCLS.DefaultMaxDownloadingTasks
                             Case 2 : Exit Sub
                         End Select
                     End If
-                    If CInt(TXT_MAX_JOBS_CHANNELS.Value) > SettingsCLS.DefaultMaxDownloadingTasks Then
+                    If CInt(TXT_MAX_JOBS_CHANNELS.Value) > SettingsCLS.DefaultMaxDownloadingTasks_Channels Then
-                        Select Case a.Invoke("channels", TXT_MAX_JOBS_CHANNELS.Value)
+                        Select Case a.Invoke("channels", SettingsCLS.DefaultMaxDownloadingTasks_Channels, TXT_MAX_JOBS_CHANNELS.Value)
-                            Case 1 : TXT_MAX_JOBS_CHANNELS.Value = SettingsCLS.DefaultMaxDownloadingTasks
+                            Case 1 : TXT_MAX_JOBS_CHANNELS.Value = SettingsCLS.DefaultMaxDownloadingTasks_Channels
                             Case 2 : Exit Sub
                         End Select
                     End If
@@ -407,7 +407,7 @@ Namespace Editors
             If Sender.DefaultButton = ADB.Refresh Then TXT_MAX_JOBS_USERS.Value = SettingsCLS.DefaultMaxDownloadingTasks
         End Sub
         Private Sub TXT_MAX_JOBS_CHANNELS_ActionOnButtonClick(ByVal Sender As ActionButton, ByVal e As EventArgs) Handles TXT_MAX_JOBS_CHANNELS.ActionOnButtonClick
-            If Sender.DefaultButton = ADB.Refresh Then TXT_MAX_JOBS_CHANNELS.Value = SettingsCLS.DefaultMaxDownloadingTasks
+            If Sender.DefaultButton = ADB.Refresh Then TXT_MAX_JOBS_CHANNELS.Value = SettingsCLS.DefaultMaxDownloadingTasks_Channels
         End Sub
         Private Sub ChangePositionControlsEnabling() Handles OPT_FILE_NAME_REPLACE.CheckedChanged, OPT_FILE_NAME_ADD_DATE.CheckedChanged
             Dim b As Boolean = OPT_FILE_NAME_ADD_DATE.Checked And OPT_FILE_NAME_ADD_DATE.Enabled

SCrawler/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2025.7.18.0")>
+<Assembly: AssemblyVersion("2025.8.1.0")>
-<Assembly: AssemblyFileVersion("2025.7.18.0")>
+<Assembly: AssemblyFileVersion("2025.8.1.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler/SettingsCLS.vb

@@ -23,6 +23,7 @@ Imports DoubleClickBehavior = SCrawler.DownloadObjects.STDownloader.DoubleClickB
 Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable
 #Region "Constants: defaults"
     Friend Const DefaultMaxDownloadingTasks As Integer = 5
+    Friend Const DefaultMaxDownloadingTasks_Channels As Integer = 1
     Friend Const TaskStackNamePornSite As String = "Porn sites"
     Friend Const Name_Node_Sites As String = "Sites"
     Private Const SitesValuesSeparator As String = ","
@@ -194,7 +195,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable
     Private ReadOnly BlackListFile As SFile = $"{SettingsFolderName}\BlackList.txt"
     Private ReadOnly UsersSettingsFile As SFile = $"{SettingsFolderName}\Users.xml"
     Private ReadOnly Property SettingsVersion As XMLValue(Of Integer)
-    Private Const SettingsVersionCurrent As Integer = 2
+    Private Const SettingsVersionCurrent As Integer = 3
     Friend ShortcutOpenFeed As New ButtonKey(Keys.F, True)
     Friend ShortcutOpenSearch As New ButtonKey(Keys.F,, True)
     Private Sub ChangeFeedOpenMode()
@@ -366,9 +367,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable
         ReparseMissingInTheRoutine = New XMLValue(Of Boolean)("ReparseMissingInTheRoutine", False, MyXML, n)
         UseDefaultAccountIfMissing = New XMLValue(Of Boolean)("UseDefaultAccountIfMissing", True, MyXML, n)
         AutomationBrushUndownloadedPlansMinutes = New XMLValue(Of Integer)("AutomationBrushUndownloadedPlansMinutes", 10080, MyXML, n)
-        DownDetectorEnabled = New XMLValue(Of Boolean)("DownDetectorEnabled", True, MyXML, n)
+        DownDetectorEnabled = New XMLValue(Of Boolean)("DownDetectorEnabled", False, MyXML, n)
-        'TODELETE: DownDetectorEnabled change
-        If SettingsVersion.Value < SettingsVersionCurrent Then DownDetectorEnabled.Value = False 'SettingsVersionCurrent = 2
 
         'Downloading: file naming
         n = {"Downloading", "FileName"}
@@ -392,7 +391,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable
         ChannelsDefaultReadyForDownload = New XMLValue(Of Boolean)("ChannelsDefaultReadyForDownload", False, MyXML, n)
         ChannelsDefaultTemporary = New XMLValue(Of Boolean)("ChannelsDefaultTemporary", True, MyXML, n)
         ChannelsHideExistsUser = New XMLValue(Of Boolean)("HideExistsUser", True, MyXML, n)
-        ChannelsMaxJobsCount = New XMLValue(Of Integer)("MaxJobsCount", DefaultMaxDownloadingTasks, MyXML, n)
+        ChannelsMaxJobsCount = New XMLValue(Of Integer)("MaxJobsCount", DefaultMaxDownloadingTasks_Channels, MyXML, n)
         n = {Name_Node_Sites, "Channels", "Users"}
         FromChannelDownloadTop = New XMLValue(Of Integer)("FromChannelDownloadTop", 10, MyXML, n)
         FromChannelDownloadTopUse = New XMLValue(Of Boolean)("FromChannelDownloadTopUse", False, MyXML, n)
@@ -497,6 +496,8 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable
         AdvancedFilter.IsViewFilter = True
         Labels.AddRange({AdvancedFilter}.GetGroupsLabels, False)
 
+        'TODELETE: DefaultMaxDownloadingTasks_Channels
+        If Not SettingsVersion = SettingsVersionCurrent Then ChannelsMaxJobsCount.Value = DefaultMaxDownloadingTasks_Channels 'SettingsVersionCurrent = 3
         SettingsVersion.Value = SettingsVersionCurrent
 
         MyXML.EndUpdate()