2024.4.14.0

Delete old notes and comments
API.Facebook: add app-id extraction from page; remove app-id from site requirements; update tokens parsing; update tokens regex
API.Instagram: add default function to parse tokens

Changelog.md

@@ -1,3 +1,24 @@
+# 2024.4.14.0
+
+*2024-04-14*
+
+- Fixed
+  - Facebook: can't get tokens
+
+# 2024.4.13.0
+
+*2024-04-13*
+
+- Added
+  - Minor improvements
+- PluginProvider
+  - IPluginContentProvider: added `ResetHistoryData` function
+- Fixed
+  - Sites
+    - TikTok: remove last download date when erasing history data
+    - YouTube: remove last download date when erasing history data
+    - Instagram: **saved posts aren't downloading**
+
 # 2024.4.10.0
 
 *2024-04-10*

SCrawler.PluginProvider/Interfaces/IPluginContentProvider.vb

@@ -40,5 +40,6 @@ Namespace Plugin
         Sub GetMedia(ByVal Token As Threading.CancellationToken)
         Sub Download(ByVal Token As Threading.CancellationToken)
         Sub DownloadSingleObject(ByVal Data As IDownloadableMedia, ByVal Token As Threading.CancellationToken)
+        Sub ResetHistoryData()
     End Interface
 End Namespace

SCrawler.PluginProvider/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2024.4.10.0")>
-<Assembly: AssemblyFileVersion("2024.4.10.0")>
+<Assembly: AssemblyVersion("2024.4.13.0")>
+<Assembly: AssemblyFileVersion("2024.4.13.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler.YouTube/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2024.4.10.0")>
-<Assembly: AssemblyFileVersion("2024.4.10.0")>
+<Assembly: AssemblyVersion("2024.4.14.0")>
+<Assembly: AssemblyFileVersion("2024.4.14.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb

@@ -709,7 +709,7 @@ Namespace API.YouTube.Objects
                     Bitrate = 0
                     _MediaType = UMTypes.Undefined
                     If SelectedVideoIndex >= 0 Then
-                        'URGENT: 2023.3.4 -> 2023.7.6
+                        '2023.3.4 -> 2023.7.6
                         'cmd.StringAppend($"bv*[format_id={SelectedVideo.ID}]")
                         cmd.StringAppend(SelectedVideo.ID)
                         _Size = SelectedVideo.Size
@@ -726,7 +726,7 @@ Namespace API.YouTube.Objects
                     End If
                     If SelectedAudioIndex >= 0 Then
                         Dim atCodec$
-                        'URGENT: 2023.3.4 -> 2023.7.6
+                        '2023.3.4 -> 2023.7.6
                         'cmd.StringAppend($"ba*[format_id={SelectedAudio.ID}]", "+")
                         cmd.StringAppend(SelectedAudio.ID, "+")
                         If OutputAudioCodec.StringToLower = ac3 Then
@@ -769,7 +769,7 @@ Namespace API.YouTube.Objects
                         subs = $"--write-subs --write-auto-subs --sub-format {OutputSubtitlesFormat.StringToLower} --sub-langs ""{subs}"" --convert-subs {OutputSubtitlesFormat.StringToLower}"
                     End If
                     If Not cmd.IsEmptyString Then
-                        'URGENT: 2023.3.4 -> 2023.7.6
+                        '2023.3.4 -> 2023.7.6
                         'cmd = $"yt-dlp -f ""{cmd}"""
                         'cmd = $"yt-dlp -f {cmd}"
                         cmd = $"{YTDLP_NAME} -f {cmd}"
@@ -1219,19 +1219,21 @@ Namespace API.YouTube.Objects
 
                                 'Subtitles
                                 ThrowAny(Token)
-                                If PostProcessing_OutputSubtitlesFormats.Count > 0 Then
+                                If SubtitlesSelectedIndexes.Count > 0 And Not OutputSubtitlesFormat.IsEmptyString Then
                                     files = SFile.GetFiles(File, String.Format(fPatternFiles, OutputSubtitlesFormat.StringToLower),, EDP.ReturnValue)
-                                    AddFile(files)
                                     If files.ListExists Then
-                                        For Each f In files
-                                            For Each format In PostProcessing_OutputSubtitlesFormats
-                                                format = format.StringToLower
-                                                commandFile = $"{f.PathWithSeparator}{f.Name}.{format}"
-                                                AddFile(commandFile)
-                                                ThrowAny(Token)
-                                                .Execute($"ffmpeg -i ""{f}"" ""{commandFile}""")
+                                        AddFile(files)
+                                        If PostProcessing_OutputSubtitlesFormats.Count > 0 Then
+                                            For Each f In files
+                                                For Each format In PostProcessing_OutputSubtitlesFormats
+                                                    format = format.StringToLower
+                                                    commandFile = $"{f.PathWithSeparator}{f.Name}.{format}"
+                                                    AddFile(commandFile)
+                                                    ThrowAny(Token)
+                                                    .Execute($"ffmpeg -i ""{f}"" ""{commandFile}""")
+                                                Next
                                             Next
-                                        Next
+                                        End If
                                     End If
                                 End If
 

SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2024.4.10.0")>
-<Assembly: AssemblyFileVersion("2024.4.10.0")>
+<Assembly: AssemblyVersion("2024.4.14.0")>
+<Assembly: AssemblyFileVersion("2024.4.14.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler/API/Base/UserDataBase.vb

@@ -1001,7 +1001,7 @@ BlockNullPicture:
 
                     x.Save(MyFileSettings)
                 End Using
-                If Not IsSavedPosts Then Settings.UpdateUsersList(User)
+                If Not IsSavedPosts Then Settings.UpdateUsersList(User, True)
             Catch ex As Exception
                 LogError(ex, "user information saving error")
             End Try
@@ -1894,7 +1894,9 @@ BlockNullPicture:
                         If m.Contains(IUserData.EraseMode.History) Then
                             If MyFilePosts.Delete(SFO.File, SFODelete.DeleteToRecycleBin, e) Then result = True
                             If MyFileData.Delete(SFO.File, SFODelete.DeleteToRecycleBin, e) Then result = True
+                            LastUpdated = Nothing
                             EraseData_AdditionalDataFiles()
+                            UpdateUserInformation()
                         End If
                         If m.Contains(IUserData.EraseMode.Data) Then
                             Dim files As List(Of SFile) = SFile.GetFiles(DownloadContentDefault_GetRootDir.CSFileP,, SearchOption.AllDirectories, e)
@@ -1916,7 +1918,7 @@ BlockNullPicture:
                 Return ErrorsDescriber.Execute(EDP.SendToLog + EDP.ReturnValue, ex, $"EraseData({CInt(Mode)}): {ToStringForLog()}", False)
             End Try
         End Function
-        Protected Overridable Sub EraseData_AdditionalDataFiles()
+        Protected Overridable Sub EraseData_AdditionalDataFiles() Implements IPluginContentProvider.ResetHistoryData
         End Sub
         Friend Overridable Function Delete(Optional ByVal Multiple As Boolean = False, Optional ByVal CollectionValue As Integer = -1) As Integer Implements IUserData.Delete
             Dim f As SFile = SFile.GetPath(MyFile.CutPath.Path)

SCrawler/API/Facebook/Declarations.vb

@@ -11,9 +11,8 @@ Imports PersonalUtilities.Functions.XML.Base
 Imports PersonalUtilities.Functions.RegularExpressions
 Namespace API.Facebook
     Friend Module Declarations
-        Friend ReadOnly Regex_UserToken_dtsg As RParams = RParams.DMS("DTSGInitialData.:.?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue)
-        Friend ReadOnly Regex_UserToken_lsd As RParams = RParams.DMS("LSD.:.?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue)
         Friend ReadOnly Regex_UserID As RParams = RParams.DMS("userid.:.(\d+)", 1, RegexOptions.IgnoreCase, EDP.ReturnValue)
+        Friend ReadOnly Regex_AppID As RParams = RParams.DMS("APP_ID.:.(\d+)", 1, RegexOptions.IgnoreCase, EDP.ReturnValue)
 
         Friend ReadOnly Regex_Photos_by As RParams = RParams.DMS("photos_by"",""id"":""([^""]+)", 1, EDP.ReturnValue)
         Friend ReadOnly Regex_FileName As RParams = RParams.DM("([^/\?]+\..{3,4})(?=(\?|\Z))", 0, EDP.ReturnValue)

SCrawler/API/Facebook/SiteSettings.vb

@@ -18,7 +18,7 @@ Namespace API.Facebook
 #Region "Auth"
         <PropertyOption(AllowNull:=False, ControlText:="Accept", ControlToolTip:="Header 'Accept'", IsAuth:=True), ControlNumber(21), PXML, PClonable>
         Friend ReadOnly Property Header_Accept As PropertyValue
-        <PropertyOption(ControlText:="x-ig-app-id", AllowNull:=True, IsAuth:=True)>
+        <PropertyOption(ControlText:="x-ig-app-id", AllowNull:=True, IsAuth:=True), HiddenControl>
         Friend Overrides ReadOnly Property HH_IG_APP_ID As PropertyValue
             Get
                 Return __HH_IG_APP_ID
@@ -74,7 +74,7 @@ Namespace API.Facebook
 #End Region
 #Region "BaseAuthExists, GetUserUrl, GetUserPostUrl, IsMyUser, IsMyImageVideo"
         Friend Overrides Function BaseAuthExists() As Boolean
-            Return Responser.CookiesExists And ACheck(HH_IG_APP_ID.Value) And CBool(DownloadData_Impl.Value)
+            Return Responser.CookiesExists And CBool(DownloadData_Impl.Value) 'And ACheck(HH_IG_APP_ID.Value)
         End Function
         Friend Overrides Function GetUserUrl(ByVal User As IPluginContentProvider) As String
             Return DirectCast(User, UserData).GetProfileUrl

SCrawler/API/Facebook/UserData.vb

@@ -136,6 +136,7 @@ Namespace API.Facebook
         Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
             If CBool(MySettings.DownloadData_Impl.Value) Then
                 Try
+                    If Responser.Headers.Value(IG.Header_IG_APP_ID).IsEmptyString Then Responser.Headers.Remove(IG.Header_IG_APP_ID)
                     ResetBaseTokens()
                     GetUserTokens(Token)
                     LoadSavePostsKV(True)
@@ -529,8 +530,14 @@ Namespace API.Facebook
                 Dim r$ = resp.GetResponse(URL)
                 If Not r.IsEmptyString Then
                     If Responser.CookiesExists Then Responser.Cookies.Update(resp.Cookies)
-                    Token_dtsg = RegexReplace(r, Regex_UserToken_dtsg)
-                    Token_lsd = RegexReplace(r, Regex_UserToken_lsd)
+                    ParseTokens(r, 0)
+                    Dim app_id$ = RegexReplace(r, Regex_AppID)
+                    If Not app_id.IsEmptyString Then
+                        If Not AEquals(Of String)(MySettings.HH_IG_APP_ID.Value, app_id) Then
+                            MySettings.HH_IG_APP_ID.Value = app_id
+                            Responser.Headers.Add(IG.Header_IG_APP_ID, app_id)
+                        End If
+                    End If
                     Token_Photosby = RegexReplace(r, Regex_Photos_by)
                     If StoryBucket.IsEmptyString Then StoryBucket = RegexReplace(r, Regex_StoryBucket)
                     If ID.IsEmptyString Then
@@ -568,14 +575,14 @@ Namespace API.Facebook
                     .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchSite, "none"))
                     .Add("Sec-Fetch-User", "?1")
                     .Add("Upgrade-Insecure-Requests", 1)
-                    Dim h$ = Responser.Headers.Value(IG.Header_Browser)
-                    If Not h.IsEmptyString Then .Add(IG.Header_Browser, h)
-                    h = Responser.Headers.Value(IG.Header_BrowserExt)
-                    If Not h.IsEmptyString Then .Add(IG.Header_BrowserExt, h)
-                    h = Responser.Headers.Value(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatform))
-                    If Not h.IsEmptyString Then .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatform, h))
-                    h = Responser.Headers.Value(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatformVersion))
-                    If Not h.IsEmptyString Then .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatformVersion, h))
+                    Dim cloneHeader As Action(Of String) = Sub(ByVal hName As String)
+                                                               Dim hValue$ = Responser.Headers.Value(hName)
+                                                               If Not hValue.IsEmptyString Then .Add(hName, hValue)
+                                                           End Sub
+                    cloneHeader.Invoke(IG.Header_Browser)
+                    cloneHeader.Invoke(IG.Header_BrowserExt)
+                    cloneHeader.Invoke(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatform).Name)
+                    cloneHeader.Invoke(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaPlatformVersion).Name)
                     .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecChUaMobile, "?0"))
                     .Add("Sec-Ch-Ua-Model", "")
                 End With

SCrawler/API/Instagram/Declarations.vb

@@ -18,6 +18,8 @@ Namespace API.Instagram
         Friend ReadOnly ObtainMedia_SizeFuncPic_RegexP As RParams = RParams.DMS("_p(\d+)x(\d+)", 1, EDP.ReturnValue)
         Friend ReadOnly ObtainMedia_SizeFuncPic_RegexS As RParams = RParams.DMS("_s(\d+)x(\d+)", 1, EDP.ReturnValue)
         Friend Const PageTokenRegexPatternDefault As String = "\[\],{""token"":""(.*?)""},\d+\]"
+        Friend ReadOnly Regex_UserToken_dtsg As RParams = RParams.DMS("DTSGInitialData["":,.\[\]]*?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue)
+        Friend ReadOnly Regex_UserToken_lsd As RParams = RParams.DMS("LSD["":,.\[\]]*?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue)
         Friend Sub UpdateResponser(ByVal Source As IResponse, ByRef Destination As Responser, ByVal UpdateWwwClaim As Boolean)
             Const r_wwwClaimName$ = "x-ig-set-www-claim"
             Const r_tokenName$ = SiteSettings.Header_CSRF_TOKEN_COOKIE

SCrawler/API/Instagram/UserData.GQL.vb

@@ -297,37 +297,51 @@ Namespace API.Instagram
                     End With
                 End With
                 Dim r$ = Responser.GetResponse(MySiteSettings.GetUserUrl(Me))
-                If Not r.IsEmptyString Then
-                    Dim rr As RParams = RParams.DM(PageTokenRegexPatternDefault, 0, RegexReturn.List, EDP.ReturnValue)
-                    Dim tokens As List(Of String) = RegexReplace(r, rr)
-                    Dim tt$, ttVal$
-                    If tokens.ListExists Then
-                        With rr
-                            .Match = Nothing
-                            .MatchSub = 1
-                            .WhatGet = RegexReturn.Value
-                        End With
-                        For Each tt In tokens
-                            If Not Token_lsd.IsEmptyString And Not Token_dtsg.IsEmptyString Then
-                                Exit For
-                            Else
-                                ttVal = RegexReplace(tt, rr)
-                                If Not ttVal.IsEmptyString Then
-                                    If ttVal.Contains(":") Then
-                                        If Token_dtsg.IsEmptyString Then Token_dtsg = ttVal
-                                    Else
-                                        If Token_lsd.IsEmptyString Then Token_lsd = ttVal
-                                    End If
-                                End If
-                            End If
-                        Next
-                    End If
-                End If
+                ParseTokens(r, 0)
             Catch ex As Exception
             Finally
                 ChangeResponserMode(_UseGQL, Not _UseGQL)
             End Try
         End Sub
+        Protected Sub ParseTokens(ByVal r As String, ByVal Attempt As Integer)
+            Try
+                If Not r.IsEmptyString Then
+                    ResetBaseTokens()
+                    Select Case Attempt
+                        Case 0
+                            Dim rr As RParams = RParams.DM(PageTokenRegexPatternDefault, 0, RegexReturn.List, EDP.ReturnValue)
+                            Dim tokens As List(Of String) = RegexReplace(r, rr)
+                            Dim tt$, ttVal$
+                            If tokens.ListExists Then
+                                With rr
+                                    .Match = Nothing
+                                    .MatchSub = 1
+                                    .WhatGet = RegexReturn.Value
+                                End With
+                                For Each tt In tokens
+                                    If Not Token_lsd.IsEmptyString And Not Token_dtsg.IsEmptyString Then
+                                        Exit For
+                                    Else
+                                        ttVal = RegexReplace(tt, rr)
+                                        If Not ttVal.IsEmptyString Then
+                                            If ttVal.Contains(":") Then
+                                                If Token_dtsg.IsEmptyString Then Token_dtsg = ttVal
+                                            Else
+                                                If Token_lsd.IsEmptyString Then Token_lsd = ttVal
+                                            End If
+                                        End If
+                                    End If
+                                Next
+                            End If
+                        Case 1
+                            Token_dtsg = RegexReplace(r, Regex_UserToken_dtsg)
+                            Token_lsd = RegexReplace(r, Regex_UserToken_lsd)
+                    End Select
+                    If Not ValidateBaseTokens() And Attempt = 0 Then ParseTokens(r, Attempt + 1)
+                End If
+            Catch
+            End Try
+        End Sub
 #End Region
     End Class
 End Namespace

SCrawler/API/Instagram/UserData.vb

@@ -886,8 +886,8 @@ NextPageBlock:
                             PostIDKV = New PostKV(.Value("code"), .Value("id"), Section)
                             PostOriginUrl = DefaultParser_PostUrlCreator(PostIDKV)
                             Pinned = .Contains("timeline_pinned_user_ids")
-                            If (Section = Sections.Timeline And Not DefaultParser_IgnorePass) AndAlso PostKvExists(PostIDKV) Then
-                                If Not Pinned Then Return False
+                            If Not DefaultParser_IgnorePass AndAlso PostKvExists(PostIDKV) Then
+                                If Not Section = Sections.Timeline OrElse Not Pinned Then Return False
                             Else
                                 _TempPostsList.Add(PostIDKV.ID)
                                 PostsKVIDs.ListAddValue(PostIDKV, LNC)
@@ -1233,6 +1233,7 @@ NextPageBlock:
         Protected Overrides Sub EraseData_AdditionalDataFiles()
             Dim f As SFile = MyFilePostsKV
             If f.Exists Then f.Delete(SFO.File, SFODelete.DeleteToRecycleBin, EDP.ReturnValue)
+            FirstLoadingDone = False
         End Sub
 #End Region
 #Region "Exceptions"

SCrawler/API/PornHub/UserData.vb

@@ -29,7 +29,6 @@ Namespace API.PornHub
         Private Const Name_DownloadFavorite As String = "DownloadFavorite"
         Private Const Name_DownloadGifs As String = "DownloadGifs"
         Private Const Name_DownloadPhotoOnlyFromModelHub As String = "DownloadPhotoOnlyFromModelHub"
-        <Obsolete> Private Const Name_IsUser As String = "IsUser"
 #End Region
 #Region "Structures"
         Private Structure FlashVar : Implements IRegExCreator
@@ -254,14 +253,7 @@ Namespace API.PornHub
                     DownloadFavorite = .Value(Name_DownloadFavorite).FromXML(Of Boolean)(False)
                     DownloadGifs = .Value(Name_DownloadGifs).FromXML(Of Integer)(False)
                     DownloadPhotoOnlyFromModelHub = .Value(Name_DownloadPhotoOnlyFromModelHub).FromXML(Of Boolean)(True)
-                    If .Contains(Name_SiteMode) Then
-                        SiteMode = .Value(Name_SiteMode).FromXML(Of Integer)(SiteModes.User)
-                    Else
-                        'TODELETE: PornHub 'IsUser' 20231113
-#Disable Warning BC40008
-                        SiteMode = IIf(.Value(Name_IsUser).FromXML(Of Boolean)(True), SiteModes.User, SiteModes.Search)
-#Enable Warning
-                    End If
+                    SiteMode = .Value(Name_SiteMode).FromXML(Of Integer)(SiteModes.User)
                     UpdateUserOptions()
                 Else
                     If UpdateUserOptions() Then .Value(Name_LabelsName) = LabelsString
@@ -404,7 +396,6 @@ Namespace API.PornHub
                 Dim r$ = Responser.GetResponse(URL)
                 If Not r.IsEmptyString Then
                     Dim l As List(Of UserVideo) = RegexFields(Of UserVideo)(r, {RegexUserVideos}, {6, 7, 3, 10})
-                    'URGENT: PornHub: changed list trimming
                     'If l.ListExists And Not SiteMode = SiteModes.Playlists Then l = l.ListTake(3, l.Count).ToList
                     If l.ListExists And Not SiteMode = SiteModes.Playlists Then l = l.ListTake(1, l.Count).ToList
                     If l.ListExists Then

SCrawler/API/ThreadsNet/SiteSettings.vb

@@ -103,8 +103,6 @@ Namespace API.ThreadsNet
 
             With Responser
                 .Accept = "*/*"
-                'URGENT: remove after debug
-                .DeclaredError = EDP.SendToLog + EDP.ThrowException
                 If .UserAgentExists Then useragent = .UserAgent
                 With .Headers
                     If .Count > 0 Then
@@ -125,6 +123,8 @@ Namespace API.ThreadsNet
                     .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchMode, "cors"))
                     .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchSite, "same-origin"))
                     .Add("Sec-Fetch-User", "?1")
+                    .Add("dht", 1)
+                    .Add("drp", 1)
                     .Add(Instagram.UserData.GQL_HEADER_FB_FRINDLY_NAME, "BarcelonaProfileThreadsTabRefetchableQuery")
                 End With
                 .CookiesExtractMode = Responser.CookiesExtractModes.Any

SCrawler/API/ThreadsNet/UserData.vb

@@ -157,38 +157,22 @@ Namespace API.ThreadsNet
         Private Function UpdateCredentials(Optional ByVal e As ErrorsDescriber = Nothing) As Boolean
             Dim URL$ = $"https://www.threads.net/@{NameTrue}"
             ResetBaseTokens()
+            Dim headers As New HttpHeaderCollection
+            headers.AddRange(Responser.Headers)
             Try
-                Responser.Method = "GET"
-                Responser.Referer = URL
-                Responser.Headers.Remove(GQL_HEADER_FB_LSD)
+                With Responser
+                    .Method = "GET"
+                    .Referer = URL
+                    With .Headers
+                        .Remove(GQL_HEADER_FB_LSD)
+                        .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchDest, "document"))
+                        .Add(HttpHeaderCollection.GetSpecialHeader(MyHeaderTypes.SecFetchMode, "navigate"))
+                    End With
+                End With
                 WaitTimer()
                 Dim r$ = Responser.GetResponse(URL,, EDP.ThrowException)
-                Dim rr As RParams
-                Dim tt$, ttVal$
                 If Not r.IsEmptyString Then
-                    rr = RParams.DM(Instagram.PageTokenRegexPatternDefault, 0, RegexReturn.List, EDP.ReturnValue)
-                    Dim tokens As List(Of String) = RegexReplace(r, rr)
-                    If tokens.ListExists Then
-                        With rr
-                            .Match = Nothing
-                            .MatchSub = 1
-                            .WhatGet = RegexReturn.Value
-                        End With
-                        For Each tt In tokens
-                            If Not Token_dtsg.IsEmptyString And Not Token_lsd.IsEmptyString Then
-                                Exit For
-                            Else
-                                ttVal = RegexReplace(tt, rr)
-                                If Not ttVal.IsEmptyString Then
-                                    If ttVal.Contains(":") Then
-                                        If Token_dtsg.IsEmptyString Then Token_dtsg = ttVal
-                                    Else
-                                        If Token_lsd.IsEmptyString Then Token_lsd = ttVal
-                                    End If
-                                End If
-                            End If
-                        Next
-                    End If
+                    ParseTokens(r, 0)
                     If ID.IsEmptyString Then ID = RegexReplace(r, RParams.DMS("""props"":\{""user_id"":""(\d+)""\},", 1, EDP.ReturnValue))
                 End If
                 Return Valid
@@ -204,6 +188,12 @@ Namespace API.ThreadsNet
                 'LogError(ex, $"failed to update some{IIf(notFound.IsEmptyString, String.Empty, $" ({notFound})")} credentials", e)
                 LogError(eex, String.Empty, e)
                 Return False
+            Finally
+                If headers.ListExists Then
+                    Responser.Headers.Clear()
+                    Responser.Headers.AddRange(headers)
+                    headers.Dispose()
+                End If
             End Try
         End Function
 #End Region

SCrawler/API/TikTok/UserData.vb

@@ -359,6 +359,11 @@ Namespace API.TikTok
             _TempMediaList.Add(m)
         End Sub
 #End Region
+#Region "EraseData"
+        Protected Overrides Sub EraseData_AdditionalDataFiles()
+            LastDownloadDate = Nothing
+        End Sub
+#End Region
 #Region "Exception"
         Protected Overrides Function DownloadingException(ByVal ex As Exception, ByVal Message As String, Optional ByVal FromPE As Boolean = False,
                                                           Optional ByVal EObj As Object = Nothing) As Integer

SCrawler/API/Xhamster/UserData.vb

@@ -280,32 +280,18 @@ Namespace API.Xhamster
                     End If
                 End If
 
-                'TODELETE: xHamster remove old container nodes attachments 
-
                 If IsSavedPosts Then
                     URL = $"https://xhamster.com/my/favorites/{IIf(IsVideo, "videos", "photos-and-galleries")}{IIf(Page = 1, String.Empty, $"/{Page}")}"
                     containerNodes.Add(If(IsVideo, {"favoriteVideoListComponent", "models"}, {"favoritesGalleriesAndPhotosCollection"}))
                 ElseIf IsChannel Then
                     URL = $"https://xhamster.com/channels/{TrueName}/newest{IIf(Page = 1, String.Empty, $"/{Page}")}"
-                    'containerNodes.Add({"trendingVideoListComponent", "models"})
-                    'containerNodes.Add({"pagesCategoryComponent", "trendingVideoListProps", "models"})
                 ElseIf SiteMode = SiteModes.Search Then
                     URL = GetNonUserUrl(Page)
                     containerNodes.Add({"searchResult", "models"})
                 ElseIf IsCreator Or SiteMode = SiteModes.Tags Or SiteMode = SiteModes.Categories Or SiteMode = SiteModes.Pornstars Then
                     URL = GetNonUserUrl(Page)
-                    'If SiteMode = SiteModes.Pornstars Then
-                    '    containerNodes.Add({"trendingVideoListComponent", "models"})
-                    '    containerNodes.Add({"pagesCategoryComponent", "trendingVideoListProps", "models"})
-                    'Else
-                    '    containerNodes.Add({"pagesCategoryComponent", "trendingVideoListProps", "models"})
-                    '    containerNodes.Add({"trendingVideoListComponent", "models"})
-                    'End If
-                    'containerNodes.Add({"trendingVideoSectionComponent", "videoModels"})
                 Else
                     URL = $"https://xhamster.com/users/{TrueName}/{IIf(IsVideo, "videos", "photos")}{IIf(Page = 1, String.Empty, $"/{Page}")}"
-                    'containerNodes.Add({If(IsVideo, "userVideoCollection", "userGalleriesCollection")})
-                    'containerNodes.Add(If(IsVideo, {"videoListComponent", "models"}, {"userGalleriesCollection"}))
                 End If
                 ThrowAny(Token)
 

SCrawler/API/YouTube/UserData.vb

@@ -467,6 +467,13 @@ Namespace API.YouTube
             _TempMediaList.Add(New UserMedia(Data))
         End Sub
 #End Region
+#Region "EraseData"
+        Protected Overrides Sub EraseData_AdditionalDataFiles()
+            LastDownloadDateVideos = Nothing
+            LastDownloadDateShorts = Nothing
+            LastDownloadDatePlaylist = Nothing
+        End Sub
+#End Region
 #Region "DownloadingException"
         Protected Overrides Function DownloadingException(ByVal ex As Exception, ByVal Message As String, Optional ByVal FromPE As Boolean = False,
                                                           Optional ByVal EObj As Object = Nothing) As Integer

SCrawler/Download/Automation/AutoDownloaderEditorForm.vb

@@ -100,7 +100,7 @@ Namespace DownloadObjects
                             users.ListAddList(DownloadGroup.GetUsers(g))
                         End Using
                     End If
-                    GroupUsersViewer.Show(users)
+                    GroupUsersViewer.Show(users, $"S {DEF_GROUP.TXT_NAME.Text}")
                     users.Clear()
                 End If
             Catch ex As Exception

SCrawler/Download/Automation/SchedulerEditorForm.vb

@@ -381,7 +381,7 @@ Namespace DownloadObjects
                                 users.ListAddList(Groups.DownloadGroup.GetUsers(.Self))
                             End If
                         End If
-                        Groups.GroupUsersViewer.Show(users)
+                        Groups.GroupUsersViewer.Show(users, $"S { .Name}")
                         users.Clear()
                     End With
                 End If

SCrawler/Download/Groups/GroupEditorForm.vb

@@ -85,7 +85,7 @@ Namespace DownloadObjects.Groups
                 If e = ShowUsersButtonKey Then
                     Using g As New GroupParameters
                         DEFS_GROUP.Get(g)
-                        GroupUsersViewer.Show(DownloadGroup.GetUsers(g))
+                        GroupUsersViewer.Show(DownloadGroup.GetUsers(g), $"{IIf(FilterMode, "F", "G")} {g.Name}")
                     End Using
                 End If
             Catch ex As Exception

SCrawler/Download/Groups/GroupListForm.vb

@@ -305,10 +305,11 @@ Namespace DownloadObjects.Groups
             Try
                 If _LatestSelected.ValueBetween(0, MyGroups.Count - 1) Then
                     Dim i%
+                    Dim n$ = String.Empty
                     Dim users As New List(Of API.Base.IUserData)
                     If Not IsViewFilter Then
                         i = Settings.Groups.IndexOf(MyGroups(_LatestSelected))
-                        If i >= 0 Then users.ListAddList(DownloadGroup.GetUsers(Settings.Groups(i)))
+                        If i >= 0 Then users.ListAddList(DownloadGroup.GetUsers(Settings.Groups(i))) : n = $"F {Settings.Groups(i).Name}"
                     ElseIf _LatestSelected.ValueBetween(0, MyGroupParams.Count - 1) Then
                         With MyGroupParams(_LatestSelected)
                             If TypeOf .Self Is AutoDownloader Then
@@ -325,14 +326,15 @@ Namespace DownloadObjects.Groups
                                             users.ListAddList(DownloadGroup.GetUsers(.Self))
                                         End If
                                     End If
+                                    n = $"S { .Name}"
                                 End With
                             ElseIf TypeOf .Self Is DownloadGroup Then
                                 i = Settings.Groups.IndexOf(.Name, .IsViewFilter)
-                                If i >= 0 Then users.ListAddList(DownloadGroup.GetUsers(Settings.Groups(i)))
+                                If i >= 0 Then users.ListAddList(DownloadGroup.GetUsers(Settings.Groups(i))) : n = $"G {Settings.Groups(i).Name}"
                             End If
                         End With
                     End If
-                    GroupUsersViewer.Show(users)
+                    GroupUsersViewer.Show(users, n)
                     users.Clear()
                 End If
             Catch ex As Exception

SCrawler/Download/Groups/GroupUsersViewer.vb

@@ -10,11 +10,16 @@ Imports SCrawler.API.Base
 Imports PersonalUtilities.Forms
 Namespace DownloadObjects.Groups
     Friend Class GroupUsersViewer : Inherits SimpleListForm(Of IUserData)
-        Friend Sub New(ByVal Users As IEnumerable(Of IUserData))
+        Friend Sub New(ByVal Users As IEnumerable(Of IUserData), ByVal AdditText As String)
             MyBase.New(Users, Settings.Design)
             DesignXMLNodeName = "GroupUsersViewer"
             Provider = New CustomProvider(Function(u As UserDataBase) u.ToStringForLog)
-            FormText = "Users"
+            FormText = $"Users ({If(Users?.Count, 0)})"
+            If Not AdditText.IsEmptyString Then
+                Dim a$ = AdditText.Trim.Take(100).ListToStringE(String.Empty,,, " ", EDP.ReturnValue)
+                If Not a = AdditText Then AdditText &= "..."
+                FormText &= $" [{a}]"
+            End If
             Icon = My.Resources.UsersIcon_32
             MyDefs.DelegateClosingChecker = False
             Mode = SimpleListFormModes.SelectedItems
@@ -23,18 +28,26 @@ Namespace DownloadObjects.Groups
         End Sub
         Protected Overrides Sub MyForm_KeyDown(sender As Object, e As KeyEventArgs)
             Try
+                Dim b As Boolean = True
                 If e.KeyCode = Keys.F And e.Control Then
-                    e.Handled = True
                     FocusUser()
+                ElseIf e.KeyCode = Keys.F3 Then
+                    EditUser(e.Alt)
                 ElseIf e = ShowUsersButtonKey Then
                     MsgBoxE(New MMessage(DataSourceCollection.ListToStringE(vbCr, Provider,,, EDP.LogMessageValue), "User list") With {.Editable = True})
+                ElseIf e.KeyCode = Keys.F1 And Not e.Alt And Not e.Control Then
+                    MsgBoxE({$"Hotkeys:{vbCr}Alt+F1 - show user list{vbCr}Ctrl+F - find the selected user in the main window{vbCr}" &
+                             $"F3 - edit selected user{vbCr}Alt+F3 - edit selected collection", "Hotkeys"})
+                Else
+                    b = False
                 End If
+                If b Then e.Handled = True
             Catch
             End Try
         End Sub
-        Friend Overloads Shared Sub Show(ByVal Users As IEnumerable(Of IUserData))
+        Friend Overloads Shared Sub Show(ByVal Users As IEnumerable(Of IUserData), ByVal AdditText As String)
             If Users.ListExists Then
-                MainFrameObj.OpenedGroupUsersForms.Add(New GroupUsersViewer(Users))
+                MainFrameObj.OpenedGroupUsersForms.Add(New GroupUsersViewer(Users, AdditText))
                 MainFrameObj.OpenedGroupUsersForms.Last.Show()
             Else
                 MsgBoxE({"No users were found based on the selected parameters", "Show group users"}, vbExclamation)
@@ -49,6 +62,15 @@ Namespace DownloadObjects.Groups
             Catch
             End Try
         End Sub
+        Private Sub EditUser(ByVal EditCollection As Boolean)
+            Try
+                If _LatestSelected.ValueBetween(0, DataSourceCollection.Count - 1) Then
+                    Dim u As IUserData = Settings.GetUser(DataSourceCollection(_LatestSelected).Key, EditCollection)
+                    If Not u Is Nothing Then ControlInvokeFast(MainFrameObj.MF, Sub() MainFrameObj.MF.EditSelectedUser(u), EDP.None)
+                End If
+            Catch
+            End Try
+        End Sub
         Friend Overloads Sub Show()
             MyForm.Show()
         End Sub

SCrawler/MainFrame.vb

@@ -1706,9 +1706,9 @@ ResumeDownloadingOperation:
             End If
         End If
     End Sub
-    Private Sub EditSelectedUser()
+    Friend Sub EditSelectedUser(Optional ByVal CUser As IUserData = Nothing)
         Const MsgTitle$ = "User update"
-        Dim user As IUserData = GetSelectedUser()
+        Dim user As IUserData = If(CUser, GetSelectedUser())
         If Not user Is Nothing Then
             On Error Resume Next
             If Not user.IsCollection OrElse DirectCast(user, UserDataBind).Count > 0 Then

SCrawler/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2024.4.10.0")>
-<Assembly: AssemblyFileVersion("2024.4.10.0")>
+<Assembly: AssemblyVersion("2024.4.14.0")>
+<Assembly: AssemblyFileVersion("2024.4.14.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler/PluginsEnvironment/Hosts/UserDataHost.vb

@@ -104,6 +104,9 @@ Namespace Plugin.Hosts
                 End With
             End If
         End Sub
+        Protected Overrides Sub EraseData_AdditionalDataFiles()
+            ExternalPlugin.ResetHistoryData()
+        End Sub
         Protected Overrides Function DownloadingException(ByVal ex As Exception, ByVal Message As String, Optional ByVal FromPE As Boolean = False,
                                                           Optional ByVal EObj As Object = Nothing) As Integer
             LogError(ex, Message)

SCrawler/SettingsCLS.vb

@@ -825,14 +825,19 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable
             Return _UserListUpdateRequired
         End Get
     End Property
-    Friend Overloads Sub UpdateUsersList(ByVal u As UserInfo)
+    Friend Overloads Sub UpdateUsersList(ByVal u As UserInfo, Optional ByVal OnlyDiff As Boolean = False)
+        Dim result As Boolean = True
         Dim i% = UsersList.IndexOf(u)
         If i >= 0 Then
-            UsersList(i) = u
+            If Not OnlyDiff OrElse Not UserInfo.ExactEquals(UsersList(i), u) Then
+                UsersList(i) = u
+            Else
+                result = False
+            End If
         Else
             UsersList.Add(u)
         End If
-        UpdateUsersList()
+        If result Then UpdateUsersList()
     End Sub
     Friend Overloads Sub UpdateUsersList()
         Try

SCrawler/UserInfo.vb

@@ -182,6 +182,22 @@ Partial Friend Module MainMod
         End Function
 #End Region
 #Region "IEquatable Support"
+        Friend Shared Function ExactEquals(ByVal x As UserInfo, ByVal y As UserInfo) As Boolean
+            Return x.Name = y.Name And
+                   x.Site = y.Site And
+                   x.Plugin = y.Plugin And
+                   x.AccountName = y.AccountName And
+                   x.File = y.File And
+                   x.IsSubscription = y.IsSubscription And
+                   x.SpecialPath = y.SpecialPath And
+                   x.SpecialCollectionPath = y.SpecialCollectionPath And
+                   x.Merged = y.Merged And
+                   x.UserModel = y.UserModel And
+                   x.CollectionName = y.CollectionName And
+                   x.CollectionModel = y.CollectionModel And
+                   x.[Protected] = y.[Protected] And
+                   AEquals(Of Date)(x.LastSeen, y.LastSeen)
+        End Function
         Friend Overloads Function Equals(ByVal Other As UserInfo) As Boolean Implements IEquatable(Of UserInfo).Equals
             Return Site.StringToLower = Other.Site.StringToLower And Name.StringToLower = Other.Name.StringToLower And
                    IsSubscription = Other.IsSubscription And (Not Plugin = PathPlugin.PluginKey Or SpecialPath = Other.SpecialPath)

Tools/NET.FrameworkVersion.ps1

@@ -1,4 +1,5 @@
-$arr=Get-ChildItem -Path Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\.NETFramework\|Select-Object Name
+try{Set-ExecutionPolicy Unrestricted CurrentUser -ErrorAction SilentlyContinue}catch{}
+$arr=Get-ChildItem -Path Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\.NETFramework\|Select-Object Name
 $mv=0
 $found=0
 foreach($v in $arr){if($v.Name -match "v4.0."){$mv=$v.Name}}