2026.3.23.0

API.EditorExchangeOptionsBase: move the 'Apply' function
API.EditorExchangeOptionsBase_P: remove the 'Apply' function
API.UserDataBase: update the 'UserSiteNameUpdate' function; remove the '_ForceSaveUserInfoOnException' property; update the 'DownloadContentDefault_ConvertWebp' function; add the 'DownloadContentDefault_ConvertWebp_TestImg' and 'DownloadContentDefault_ConvertWebp_Impl' functions
API.Bluesky: add the 'EditorExchangeOptions' class; add 'did:' handle for recognizing
API.PornHub: update M3U8 download functions
API.TikTok: fix where gallery-dl would hang when downloading stories
API.Twitter: fix downloading a single media file (standalone downloader) fails
API.Xhamster: add 'webm' to 'jpg' correct conversion
FeedMedia: move the 'ImageRenderer2' class into the 'UserImage' class
UserImage: add the 'ImageRenderer2' class

Changelog.md

@@ -2,13 +2,51 @@
 - [ffmpeg](https://github.com/AAndyProgram/SCrawler/wiki/Settings#ffmpeg)
   - x64 version - [release](https://github.com/GyanD/codexffmpeg/releases/tag/5.1.2); [zip](https://github.com/GyanD/codexffmpeg/releases/download/5.1.2/ffmpeg-5.1.2-full_build.zip); **version `5.1.2-full_build-www.gyan.dev`**
   - x86 version - [release](https://github.com/yt-dlp/FFmpeg-Builds/releases/tag/autobuild-2022-11-30-12-57); [zip](https://github.com/yt-dlp/FFmpeg-Builds/releases/download/autobuild-2022-11-30-12-57/ffmpeg-N-109274-gd7a5f068c2-win32-gpl.zip); **version `N-109457-geeb280f351-20221226`**
-- [Gallery-dl](https://github.com/AAndyProgram/SCrawler/wiki/Settings#gallery-dl) - **1.31.4**
-- [YT-DLP](https://github.com/AAndyProgram/SCrawler/wiki/Settings#yt-dlp) - **2025.12.08**
+- [Gallery-dl](https://github.com/AAndyProgram/SCrawler/wiki/Settings#gallery-dl) - **1.31.9**
+- [YT-DLP](https://github.com/AAndyProgram/SCrawler/wiki/Settings#yt-dlp) - **2026.03.17**
 - [Deno](https://github.com/AAndyProgram/SCrawler/wiki/Settings#deno) - latest *(`2.0.0` or higher)*
 - [OF-Scraper](https://github.com/AAndyProgram/SCrawler/wiki/Settings#of-scraper) - **3.12.9** ([release](https://github.com/datawhores/OF-Scraper/releases/tag/3.12.9))
 
 # 2026
 
+## 2026.3.23.0
+
+*2026-03-23*
+
+- Added
+  - Sites:
+    - Bluesky:
+      - download models (user options), like in Twitter
+      - `did` handle for recognizing
+    - PornHub: `M3U8` download functionality update
+    - xHamster: add `webm` to `jpg` correct conversion
+  - Minor improvements
+- Updated
+  - gallery-dl up to version **1.31.9**
+  - yt-dlp up to version **2026.03.17**
+- Fixed
+  - Sites:
+    - TikTok: gallery-dl downloads stories slowly when the user has no stories
+    - Twitter: downloading a single media file (standalone downloader) fails
+  - Minor bugs
+
+## 2026.2.14.0
+
+*2026-02-14*
+
+- Added
+  - Sites:
+    - Twitter: get a new username based on the user ID
+  - Minor improvements
+- Updated
+  - gallery-dl up to version **1.31.6**
+  - yt-dlp up to version **2026.02.04.233607**
+- Fixed
+  - Sites:
+    - **Instagram: some profiles aren't downloading**
+    - xHamster: videos aren't downloading
+  - Minor bugs
+
 ## 2026.1.24.0
 
 *2026-01-24*

SCrawler/API/Base/EditorExchangeOptionsBase.vb

@@ -42,5 +42,9 @@ Namespace API.Base
                 u.DownloadTextSpecialFolder = DownloadTextSpecialFolder
             End If
         End Sub
+        Friend Overridable Sub Apply(ByRef u As UserDataBase)
+            ApplyBase(u)
+            If TypeOf u Is IPSite And TypeOf Me Is IPSite Then DirectCast(u, IPSite).QueryString = DirectCast(Me, IPSite).QueryString
+        End Sub
     End Class
 End Namespace

SCrawler/API/Base/EditorExchangeOptionsBase_P.vb

@@ -31,10 +31,6 @@ Namespace API.Base
             MyBase.New(s)
             DisableBase()
         End Sub
-        Friend Overridable Sub Apply(ByRef u As IPSite)
-            ApplyBase(u)
-            u.QueryString = QueryString
-        End Sub
         Protected Overridable Sub DisableBase()
             _ApplyBase_Name = False
             _ApplyBase_Text = False

SCrawler/API/Base/TokenBatch.vb

@@ -13,12 +13,17 @@ Namespace API.Base
         Friend Property TempPostsList As List(Of String)
         Protected ReadOnly Token As CancellationToken
         Friend Property DebugMode As Boolean = False
+        Friend Overridable Property MyWorkingDirectory As SFile = Nothing
         Friend Sub New(ByVal _Token As CancellationToken, Optional ByVal _MainProcessName As String = Nothing, Optional ByVal WorkingDir As SFile = Nothing)
             MyBase.New(True)
             Token = _Token
             MainProcessName = _MainProcessName
             If Not WorkingDir.IsEmptyString Then ChangeDirectory(WorkingDir)
         End Sub
+        Public Overrides Sub ChangeDirectory(ByVal Directory As SFile)
+            MyBase.ChangeDirectory(Directory)
+            If Not Directory.IsEmptyString Then MyWorkingDirectory = Directory
+        End Sub
         Protected Overrides Function Internal_Execute(ByVal Commands As IEnumerable(Of String), ByVal e As ErrorsDescriber) As Boolean
             If Not Encoding.HasValue Then Encoding = UnicodeEncoding
             Return MyBase.Internal_Execute(Commands, e)

SCrawler/API/Base/UserDataBase.vb

@@ -360,8 +360,8 @@ Namespace API.Base
                 Me._UserSiteName = _UserSiteName
             End Set
         End Property
-        Protected Sub UserSiteNameUpdate(ByVal NewName As String)
-            If Not NewName.IsEmptyString And (UserSiteName.IsEmptyString Or Settings.UpdateUserSiteNameEveryTime) Then UserSiteName = NewName
+        Protected Sub UserSiteNameUpdate(ByVal NewName As String, Optional ByVal Force As Boolean = False)
+            If Not NewName.IsEmptyString And (UserSiteName.IsEmptyString Or Settings.UpdateUserSiteNameEveryTime Or Force) Then UserSiteName = NewName
         End Sub
         Friend ReadOnly Property UserModel As UsageModel Implements IUserData.UserModel
             Get
@@ -1030,7 +1030,7 @@ BlockNullPicture:
             End Try
         End Sub
         Private Sub UpdateUserInformation_Ex()
-            If _ForceSaveUserInfoOnException Then UpdateUserInformation()
+            If _ForceSaveUserInfo Then UpdateUserInformation()
         End Sub
         Friend Overridable Overloads Sub UpdateUserInformation() Implements IUserData.UpdateUserInformation
             UpdateUserInformation(False)
@@ -1207,7 +1207,6 @@ BlockNullPicture:
         Protected UseClientTokens As Boolean = False
         Protected _ForceSaveUserData As Boolean = False
         Protected _ForceSaveUserInfo As Boolean = False
-        Protected _ForceSaveUserInfoOnException As Boolean = False
         Private _DownloadInProgress As Boolean = False
         Private _EnvirUserExists As Boolean
         Private _EnvirUserSuspended As Boolean
@@ -1227,7 +1226,6 @@ BlockNullPicture:
             _DescriptionEveryTime = Settings.UpdateUserDescriptionEveryTime
             _ForceSaveUserData = False
             _ForceSaveUserInfo = False
-            _ForceSaveUserInfoOnException = False
             _EnvirUserExists = UserExists
             _EnvirUserSuspended = UserSuspended
             _EnvirCreatedByChannel = CreatedByChannel
@@ -1244,8 +1242,8 @@ BlockNullPicture:
                 Select Case Caller
                     Case NameOf(UserExists) : If Not _EnvirUserExists = CBool(NewValue) Then _EnvirChanged = True : _EnvirInvokeUserUpdated = True
                     Case NameOf(UserSuspended) : If Not _EnvirUserSuspended = CBool(NewValue) Then _EnvirChanged = True : _EnvirInvokeUserUpdated = True
-                    Case NameOf(NameTrue) : _EnvirChanged = True : _ForceSaveUserInfo = True : _ForceSaveUserInfoOnException = True
-                    Case NameOf(ID) : _EnvirChanged = True : _ForceSaveUserInfo = True : _ForceSaveUserInfoOnException = True
+                    Case NameOf(NameTrue) : _EnvirChanged = True : _ForceSaveUserInfo = True
+                    Case NameOf(ID) : _EnvirChanged = True : _ForceSaveUserInfo = True
                     Case Else : _EnvirChanged = True
                 End Select
             End If
@@ -1843,7 +1841,8 @@ BlockNullPicture:
 
                                         updateDownCount(False)
 
-                                        v.File = DownloadContentDefault_ConvertWebp(ChangeFileNameByProvider(f, v), postProcessWebp)
+                                        v.File = ChangeFileNameByProvider(f, v)
+                                        v.File = DownloadContentDefault_ConvertWebp(v, postProcessWebp)
                                         v.State = UStates.Downloaded
                                         DownloadContentDefault_PostProcessing(v, f, Token)
                                         If UseMD5Comparison And (v.Type = UTypes.GIF Or v.Type = UTypes.Picture) Then
@@ -1943,7 +1942,11 @@ stxt:
         End Function
         Protected Overridable Sub DownloadContentDefault_PostProcessing(ByRef m As UserMedia, ByVal File As SFile, ByVal Token As CancellationToken)
         End Sub
-        Protected Overridable Function DownloadContentDefault_ConvertWebp(ByVal WebpFile As SFile, ByVal Process As Boolean) As SFile
+        Protected Overridable Function DownloadContentDefault_ConvertWebp(ByVal m As UserMedia, ByVal Process As Boolean) As SFile
+            Return DownloadContentDefault_ConvertWebp_Impl(m, Process)
+        End Function
+        Protected Overridable Function DownloadContentDefault_ConvertWebp_Impl(ByVal m As UserMedia, ByVal Process As Boolean) As SFile
+            Dim WebpFile As SFile = m.File
             Dim f As SFile = WebpFile
             If Process AndAlso f.Exists Then
                 f.Path = $"{f.PathWithSeparator}Sources"
@@ -1959,6 +1962,24 @@ stxt:
             End If
             Return f
         End Function
+        Protected Function DownloadContentDefault_ConvertWebp_TestImg(ByVal m As UserMedia, ByVal Process As Boolean) As SFile
+            If m.Type = UTypes.Picture And Settings.DownloadNativeImageFormat Then
+                Using testImg As New UserImage.ImageRenderer2(m.File, EDP.ReturnValue)
+                    If testImg.IsWebP Then
+                        Dim f As SFile = m.File
+                        If f.Extension.IsEmptyString OrElse Not f.Extension = UserImage.ExtWebp Then
+                            f.Extension = UserImage.ExtWebp
+                            f = SFile.Rename(m.File, f,, EDP.ReturnValue).IfNullOrEmpty(m.File)
+                        End If
+                        m.File = f
+                        Return DownloadContentDefault_ConvertWebp_Impl(m, True).IfNullOrEmpty(f)
+                    End If
+                End Using
+            ElseIf Process Then
+                Return DownloadContentDefault_ConvertWebp_Impl(m, Process)
+            End If
+            Return m.File
+        End Function
         Protected Overridable Function DownloadContentDefault_ProcessDownloadException() As Boolean
             Return True
         End Function

SCrawler/API/Bluesky/EditorExchangeOptions.vb

@@ -0,0 +1,44 @@
+' Copyright (C) Andy https://github.com/AAndyProgram
+' This program is free software: you can redistribute it and/or modify
+' it under the terms of the GNU General Public License as published by
+' the Free Software Foundation, either version 3 of the License, or
+' (at your option) any later version.
+'
+' This program is distributed in the hope that it will be useful,
+' but WITHOUT ANY WARRANTY
+Imports SCrawler.API.Base
+Imports SCrawler.Plugin.Attributes
+Namespace API.Bluesky
+    Friend Class EditorExchangeOptions : Inherits Base.EditorExchangeOptionsBase
+        Friend Overrides Property SiteKey As String = BlueskySiteKey
+        <PSetting(NameOf(SiteSettings.DownloadModelMedia), NameOf(MySettings), Address:=SettingAddress.User)>
+        Friend Overridable Property DownloadModelMedia As Boolean = False
+        <PSetting(NameOf(SiteSettings.DownloadModelProfile), NameOf(MySettings), Address:=SettingAddress.User)>
+        Friend Overridable Property DownloadModelProfile As Boolean = False
+        Private ReadOnly Property MySettings As Object
+        Friend Sub New(ByVal s As SiteSettings)
+            DownloadModelMedia = s.DownloadModelMedia.Value
+            DownloadModelProfile = s.DownloadModelProfile.Value
+            MySettings = s
+        End Sub
+        Friend Sub New(ByVal u As UserData)
+            DownloadModelMedia = u.DownloadModelMedia
+            DownloadModelProfile = u.DownloadModelProfile
+            MySettings = u.HOST.Source
+        End Sub
+        Friend Overrides Sub Apply(ByRef u As UserDataBase)
+            MyBase.Apply(u)
+            If Not DownloadModelMedia And Not DownloadModelProfile Then
+                DownloadModelMedia = True
+            ElseIf DownloadModelMedia Then
+                DownloadModelProfile = False
+            Else
+                DownloadModelMedia = False
+            End If
+            With DirectCast(u, UserData)
+                .DownloadModelMedia = DownloadModelMedia
+                .DownloadModelProfile = DownloadModelProfile
+            End With
+        End Sub
+    End Class
+End Namespace

SCrawler/API/Bluesky/SiteSettings.vb

@@ -26,6 +26,10 @@ Namespace API.Bluesky
         <PXML> Friend ReadOnly Property TokenUpdateTime As PropertyValue
         <PropertyOption(ControlText:="Token update", ControlToolTip:="Token refresh interval (in minutes)." & vbCr & "Default: 120.", IsAuth:=True), PXML, PClonable, HiddenControl>
         Friend ReadOnly Property TokenRefreshInterval As PropertyValue
+        <PropertyOption(ControlText:="Download model 'Media'", ControlToolTip:="Parse the 'Media' block", Category:=DeclaredNames.CAT_UserDefs), PXML, PClonable>
+        Friend ReadOnly Property DownloadModelMedia As PropertyValue
+        <PropertyOption(ControlText:="Download model 'Profile'", ControlToolTip:="Parse the 'Posts' block", Category:=DeclaredNames.CAT_UserDefs), PXML, PClonable>
+        Friend ReadOnly Property DownloadModelProfile As PropertyValue
         Friend Sub New(ByVal AccName As String, ByVal Temp As Boolean)
             MyBase.New("Bluesky", "bsky.app", AccName, Temp, My.Resources.SiteResources.BlueskyIcon_32, My.Resources.SiteResources.BlueskyPic_32)
 
@@ -38,11 +42,14 @@ Namespace API.Bluesky
             TokenUpdateTime = New PropertyValue(Now.AddYears(-1))
             TokenRefreshInterval = New PropertyValue(120)
 
+            DownloadModelMedia = New PropertyValue(True)
+            DownloadModelProfile = New PropertyValue(False)
+
             _AllowUserAgentUpdate = False
             UrlPatternUser = "https://bsky.app/profile/{0}"
             ImageVideoContains = "bsky.app"
             UserRegex = RParams.DMS("bsky.app/profile/([^/\?]+)", 1, EDP.ReturnValue)
-            UserOptionsType = GetType(EditorExchangeOptionsBase)
+            UserOptionsType = GetType(EditorExchangeOptions)
         End Sub
         Protected Overrides Function UserOptionsValid(ByVal Options As Object) As Boolean
             Return DirectCast(Options, EditorExchangeOptionsBase).SiteKey = BlueskySiteKey
@@ -96,5 +103,10 @@ Namespace API.Bluesky
                 _TokenUpdating = False
             End Try
         End Function
+        Friend Overrides Function IsMyUser(ByVal UserURL As String) As ExchangeOptions
+            Dim e As ExchangeOptions = MyBase.IsMyUser(UserURL)
+            If Not e.UserName.IsEmptyString AndAlso e.UserName.StartsWith("did:") Then e.UserName = e.UserName.Replace(":", "@")
+            Return e
+        End Function
     End Class
 End Namespace

SCrawler/API/Bluesky/UserData.vb

@@ -16,6 +16,11 @@ Imports UTypes = SCrawler.API.Base.UserMedia.Types
 Imports UStates = SCrawler.API.Base.UserMedia.States
 Namespace API.Bluesky
     Friend Class UserData : Inherits UserDataBase
+#Region "XML names"
+        Private Const Name_DownloadModelMedia As String = "DownloadModelMedia"
+        Private Const Name_DownloadModelProfile As String = "DownloadModelProfile"
+        Private Const Name_ForceParseProfileInfo As String = "ForceParseProfileInfo"
+#End Region
 #Region "Declarations"
         Private ReadOnly Property MySettings As SiteSettings
             Get
@@ -28,16 +33,45 @@ Namespace API.Bluesky
             End Get
         End Property
         Private ReadOnly _TmpPosts2 As List(Of String)
+        Friend Property DownloadModelMedia As Boolean = True
+        Friend Property DownloadModelProfile As Boolean = False
+        Private Property ForceParseProfileInfo As Boolean = False
 #End Region
 #Region "Loader"
+        Private Sub UpdateUserOptions()
+            If ID.IsEmptyString AndAlso Not Name.IsEmptyString AndAlso Name.StartsWith("did@") Then
+                NameTrue = Name.Replace("@", ":")
+                ID = NameTrue
+                ForceParseProfileInfo = True
+            End If
+        End Sub
         Protected Overrides Sub LoadUserInformation_OptionalFields(ByRef Container As XmlFile, ByVal Loading As Boolean)
+            With Container
+                If Loading Then
+                    If .Contains(Name_DownloadModelMedia) Then
+                        DownloadModelMedia = .Value(Name_DownloadModelMedia).FromXML(Of Boolean)(True)
+                        DownloadModelProfile = .Value(Name_DownloadModelProfile).FromXML(Of Boolean)(False)
+                    Else
+                        DownloadModelMedia = ParseUserMediaOnly
+                        DownloadModelProfile = Not ParseUserMediaOnly
+                    End If
+                    ForceParseProfileInfo = .Value(Name_ForceParseProfileInfo).FromXML(Of Boolean)(False)
+                Else
+                    If ID.IsEmptyString Then
+                        UpdateUserOptions()
+                        .Value(Name_UserID) = ID
+                    End If
+                    .Add(Name_DownloadModelMedia, DownloadModelMedia.BoolToInteger)
+                    .Add(Name_DownloadModelProfile, DownloadModelProfile.BoolToInteger)
+                    .Add(Name_ForceParseProfileInfo, ForceParseProfileInfo.BoolToInteger)
+                End If
+            End With
         End Sub
         Friend Overrides Function ExchangeOptionsGet() As Object
-            Return New EditorExchangeOptionsBase(Me) With {.SiteKey = BlueskySiteKey}
+            Return New EditorExchangeOptions(Me)
         End Function
         Friend Overrides Sub ExchangeOptionsSet(ByVal Obj As Object)
-            If Not Obj Is Nothing AndAlso TypeOf Obj Is EditorExchangeOptionsBase AndAlso
-               DirectCast(Obj, EditorExchangeOptionsBase).SiteKey = BlueskySiteKey Then DirectCast(Obj, EditorExchangeOptionsBase).ApplyBase(Me)
+            If Not Obj Is Nothing AndAlso TypeOf Obj Is EditorExchangeOptions Then DirectCast(Obj, EditorExchangeOptions).Apply(Me)
         End Sub
 #End Region
 #Region "Initializer"
@@ -66,6 +100,13 @@ Namespace API.Bluesky
         Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
             _TmpPosts2.Clear()
             Try
+                If Not DownloadModelMedia And Not DownloadModelProfile Then
+                    DownloadModelMedia = True
+                ElseIf DownloadModelMedia Then
+                    DownloadModelProfile = False
+                Else
+                    DownloadModelMedia = False
+                End If
                 If Not CBool(MySettings.CookiesEnabled.Value) Then Responser.Cookies.Clear()
                 UpdateToken(, True)
                 _TokenUpdateCount = 0
@@ -79,7 +120,7 @@ Namespace API.Bluesky
         Private Overloads Sub DownloadData(ByVal Cursor As String, ByVal Token As CancellationToken)
             Dim URL$ = String.Empty
             Try
-                If Not IsSavedPosts And ID.IsEmptyString Then GetProfileInfo(Token)
+                If (Not IsSavedPosts And ID.IsEmptyString) Or ForceParseProfileInfo Then GetProfileInfo(Token)
                 If Not IsSavedPosts And ID.IsEmptyString Then Throw New ArgumentNullException("ID", "ID is null")
                 If UpdateToken() Then
                     Dim nextCursor$ = String.Empty
@@ -91,7 +132,9 @@ Namespace API.Bluesky
                         n = {"bookmarks"}
                         p = {"item"}
                     Else
-                        URL = $"https://bsky.social/xrpc/app.bsky.feed.getAuthorFeed?actor={ID_Encoded}&filter=posts_and_author_threads&includePins=false&limit=99"
+                        'posts_and_author_threads
+                        'posts_with_media
+                        URL = $"https://bsky.social/xrpc/app.bsky.feed.getAuthorFeed?actor={ID_Encoded}&filter={IIf(DownloadModelMedia, "posts_with_media", "posts_and_author_threads")}&includePins=false&limit=99"
                         If Not Cursor.IsEmptyString Then URL &= $"&cursor={SymbolsConverter.ASCII.EncodeSymbolsOnly(Cursor)}"
                         n = {"feed"}
                         p = {"post"}
@@ -106,7 +149,7 @@ Namespace API.Bluesky
                                     If .ListExists Then
                                         For Each post As EContainer In .Self
                                             With post(p)
-                                                c = DefaultParser(.Self,, nextCursor)
+                                                c = DefaultParser(.Self) ',, nextCursor)
                                                 Select Case c
                                                     Case CInt(DateResult.Skip) * -1 : Continue For
                                                     Case CInt(DateResult.Exit) * -1 : Exit Sub
@@ -238,6 +281,7 @@ Namespace API.Bluesky
 #Region "GetProfileInfo"
         Private Sub GetProfileInfo(ByVal Token As CancellationToken)
             Try
+                If ForceParseProfileInfo Then ForceParseProfileInfo = False : _ForceSaveUserInfo = True
                 If UpdateToken() Then
                     Dim r$ = Responser.GetResponse($"https://bsky.social/xrpc/app.bsky.actor.getProfile?actor={ID.IfNullOrEmpty(NameTrue)}")
                     TokenUpdateCountReset()
@@ -344,6 +388,9 @@ Namespace API.Bluesky
         Protected Overrides Function DownloadM3U8(ByVal URL As String, ByVal Media As UserMedia, ByVal DestinationFile As SFile, ByVal Token As CancellationToken) As SFile
             Return M3U8.Download(URL, DestinationFile, Token, Progress, Not IsSingleObjectDownload)
         End Function
+        Protected Overrides Function DownloadContentDefault_ConvertWebp(ByVal m As UserMedia, ByVal Process As Boolean) As SFile
+            Return DownloadContentDefault_ConvertWebp_TestImg(m, Process)
+        End Function
 #End Region
 #Region "DownloadSingleObject"
         Protected Overrides Sub DownloadSingleObject_GetPosts(ByVal Data As IYouTubeMediaContainer, ByVal Token As CancellationToken)

SCrawler/API/Instagram/Declarations.vb

@@ -20,6 +20,7 @@ Namespace API.Instagram
         Friend Const PageTokenRegexPatternDefault As String = "\[\],{""token"":""(.*?)""},\d+\]"
         Friend ReadOnly Regex_UserToken_dtsg As RParams = RParams.DMS("DTSGInitialData["":,.\[\]]*?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue)
         Friend ReadOnly Regex_UserToken_lsd As RParams = RParams.DMS("LSD["":,.\[\]]*?{\s*.token.:\s*""([^""]+)", 1, EDP.ReturnValue)
+        Friend ReadOnly Regex_ProfileID As RParams = RParams.DMS("profilePage_(\d+)", 1, EDP.ReturnValue)
         Friend Sub UpdateResponser(ByVal Source As IResponse, ByRef Destination As Responser, ByVal UpdateWwwClaim As Boolean)
             Const r_wwwClaimName$ = "x-ig-set-www-claim"
             Const r_tokenName$ = SiteSettings.Header_CSRF_TOKEN_COOKIE

SCrawler/API/Instagram/EditorExchangeOptions.vb

@@ -36,6 +36,8 @@ Namespace API.Instagram
         <PSetting(Caption:="Place the extracted image into the video folder")>
         Friend Property PutImageVideoFolder As Boolean
         Friend Overrides Property UserName As String
+        <PSetting(Address:=SettingAddress.User, Caption:="Verified profile", ToolTip:="This profile has a verified mark")>
+        Friend Property IsVerifiedProfile As Boolean = False
         <PSetting(Address:=SettingAddress.User, Caption:="Force update UserName", ToolTip:="Try to force update UserName if it is not found on the site")>
         Friend Property ForceUpdateUserName As Boolean = False
         <PSetting(Address:=SettingAddress.User, Caption:="Force update user information")>
@@ -57,6 +59,8 @@ Namespace API.Instagram
 
                 PutImageVideoFolder = .PutImageVideoFolder
 
+                IsVerifiedProfile = .IsVerifiedProfile
+
                 ForceUpdateUserName = .ForceUpdateUserName
                 ForceUpdateUserInfo = .ForceUpdateUserInfo
             End With

SCrawler/API/Instagram/SiteSettings.vb

@@ -148,6 +148,8 @@ Namespace API.Instagram
 #End Region
         <PropertyOption(ControlText:="Use GraphQL to download", IsAuth:=True), PXML, PClonable>
         Friend ReadOnly Property USE_GQL As PropertyValue
+        <PropertyOption(ControlText:="Use GraphQL to download user data", IsAuth:=True), PXML, PClonable, HiddenControl>
+        Friend ReadOnly Property USE_GQL_UserData As PropertyValue
 #End Region
 #Region "Download data"
         <PropertyOption(ControlText:="Download timeline", Category:=CAT_DOWN), PXML, PClonable>
@@ -165,6 +167,14 @@ Namespace API.Instagram
         <PropertyOption(ControlText:="Download tagged posts", Category:=CAT_DOWN), PXML, PClonable>
         Friend ReadOnly Property DownloadTagged As PropertyValue
         <PXML> Private ReadOnly Property DownloadTagged_Def As PropertyValue
+        <PropertyOption(ControlText:="Number of posts (verified)", ControlToolTip:="The number of posts received per request if the profile has a verified mark", Category:=CAT_DOWN), PXML, PClonable, HiddenControl>
+        Friend ReadOnly Property PostNumberVerified As PropertyValue
+        <Provider(NameOf(PostNumberVerified), FieldsChecker:=True)>
+        Private ReadOnly Property PostNumberVerifiedProvider As IFormatProvider
+        <PropertyOption(ControlText:="Number of posts (unverified)", ControlToolTip:="The number of posts received per request if the profile doesn't have a verified mark", Category:=CAT_DOWN), PXML, PClonable, HiddenControl>
+        Friend ReadOnly Property PostNumberVerifiedNot As PropertyValue
+        <Provider(NameOf(PostNumberVerifiedNot), FieldsChecker:=True)>
+        Private ReadOnly Property PostNumberVerifiedNotProvider As IFormatProvider
 #End Region
 #Region "Timers"
         Friend Const TimersUrgentTip As String = vbCr & "It is highly recommended not to change the default value."
@@ -485,6 +495,7 @@ Namespace API.Instagram
             HH_IG_WWW_CLAIM_USE_DEFAULT_ALGO = New PropertyValue(True)
             TokenUpdateIntervalProvider = New TokenRefreshIntervalProvider
             USE_GQL = New PropertyValue(False)
+            USE_GQL_UserData = New PropertyValue(True)
 
             DownloadTimeline = New PropertyValue(True)
             DownloadTimeline_Def = New PropertyValue(DownloadTimeline.Value, GetType(Boolean))
@@ -496,6 +507,10 @@ Namespace API.Instagram
             DownloadStoriesUser_Def = New PropertyValue(DownloadStoriesUser.Value, GetType(Boolean))
             DownloadTagged = New PropertyValue(False)
             DownloadTagged_Def = New PropertyValue(DownloadTagged.Value, GetType(Boolean))
+            PostNumberVerified = New PropertyValue(50)
+            PostNumberVerifiedProvider = New TimersChecker(12)
+            PostNumberVerifiedNot = New PropertyValue(12)
+            PostNumberVerifiedNotProvider = New TimersChecker(12)
 
             RequestsWaitTimer_Any = New PropertyValue(1000)
             RequestsWaitTimer_AnyProvider = New TimersChecker(0)

SCrawler/API/Instagram/UserData.GQL.vb

@@ -6,12 +6,13 @@
 '
 ' This program is distributed in the hope that it will be useful,
 ' but WITHOUT ANY WARRANTY
+Imports System.Security.Cryptography
 Imports System.Threading
-Imports SCrawler.API.Base
-Imports PersonalUtilities.Functions.XML
 Imports PersonalUtilities.Functions.RegularExpressions
+Imports PersonalUtilities.Functions.XML
 Imports PersonalUtilities.Tools.Web.Clients
 Imports PersonalUtilities.Tools.Web.Documents.JSON
+Imports SCrawler.API.Base
 Namespace API.Instagram
     Partial Friend Class UserData
 #Region "Tokens"
@@ -43,9 +44,9 @@ Namespace API.Instagram
         Private Const GQL_UserStories_DocId As String = "25231722019806941"
         Private Const GQL_UserStories_FbFriendlyName As String = "PolarisStoriesV3ReelPageStandaloneQuery"
 
-        Private Const GQL_Timeline_DocId As String = "7268577773270422"
+        Private Const GQL_Timeline_DocId As String = "7268577773270422" '"34579740524958711" '"7268577773270422"
         Private Const GQL_Timeline_FbFriendlyName As String = "PolarisProfilePostsQuery"
-        Private Const GQL_Timeline_DocId_Second As String = "7286316061475375"
+        Private Const GQL_Timeline_DocId_Second As String = "7286316061475375" '"33944389991841132" '"7286316061475375"
         Private Const GQL_Timeline_FbFriendlyName_Second As String = "PolarisProfilePostsTabContentQuery_connection"
 
         Private Const GQL_Reels_DocId As String = "7191572580905225"
@@ -64,33 +65,42 @@ Namespace API.Instagram
             Responser.Headers.Add(GQL_HEADER_FB_FRINDLY_NAME, HeaderValue)
             Responser.Headers.Add(GQL_HEADER_FB_LSD, Token_lsd)
         End Sub
-        <Obsolete("Use 'GET' function: 'GetUserData'", False)>
-        Private Sub GetUserDataGQL(ByVal Token As CancellationToken)
+        '<Obsolete("Use 'GET' function: 'GetUserData'", False)>
+        Private Function GetUserDataGQL(ByVal Token As CancellationToken) As String
             Dim vars$ = String.Format(GQL_URL_PATTERN_VARS, GQL_UserData_DocId, Token_lsd, Token_dtsg_Var, GQL_UserData_FbFriendlyName,
                                       SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & $"""id"":""{ID}"",""relay_header"":false,""render_surface"":""PROFILE""" & "}"))
             UpdateRequestNumber()
             ChangeResponserMode(True)
             UpdateHeadersGQL(GQL_UserData_FbFriendlyName)
             Dim r$ = Responser.GetResponse(GQL_URL, vars)
-            If Not r.IsEmptyString Then
-                Using j As EContainer = JsonDocument.Parse(r)
-                    If j.ListExists Then
-                        With j({"data", "user"})
-                            If .ListExists Then
-                                UserSiteName = .Value("full_name").IfNullOrEmpty(UserSiteName)
-                                Dim f As New SFile With {.Path = DownloadContentDefault_GetRootDir(), .Name = "ProfilePicture", .Extension = "jpg"}
-                                Dim pic$ = .Value({"hd_profile_pic_url_info"}, "url").IfNullOrEmpty(.Value("profile_pic_url"))
-                                If Not pic.IsEmptyString Then GetWebFile(pic, f, EDP.ReturnValue)
-                                UserDescriptionUpdate(.Value("biography"))
-                            End If
-                        End With
-                    End If
-                End Using
-            End If
-        End Sub
+            Return r
+            'If Not r.IsEmptyString Then
+            '    Using j As EContainer = JsonDocument.Parse(r)
+            '        If j.ListExists Then
+            '            With j({"data", "user"})
+            '                If .ListExists Then
+            '                    UserSiteName = .Value("full_name").IfNullOrEmpty(UserSiteName)
+            '                    IsVerifiedProfile = .Value("is_verified").FromXML(Of Boolean)(False)
+            '                    IsVerifiedProfile_Checked = True
+            '                    Dim descr$ = .Value("biography")
+            '                    If If(.Item("bio_links")?.Count, 0) > 0 Then descr.StringAppend(.Item("bio_links").Select(Function(bl) bl.Value("url")).ListToString(vbNewLine), vbNewLine)
+            '                    Dim eUrl$ = .Value("external_url")
+            '                    If Not eUrl.IsEmptyString AndAlso (descr.IsEmptyString OrElse Not descr.Contains(eUrl)) Then descr.StringAppendLine(eUrl)
+            '                    UserDescriptionUpdate(descr)
+
+            '                    Dim f As New SFile With {.Path = DownloadContentDefault_GetRootDir(), .Name = "ProfilePicture", .Extension = "jpg"}
+            '                    Dim pic$ = .Value({"hd_profile_pic_url_info"}, "url").IfNullOrEmpty(.Value("profile_pic_url"))
+            '                    If Not pic.IsEmptyString Then GetWebFile(pic, f, EDP.ReturnValue)
+            '                End If
+            '            End With
+            '        End If
+            '    End Using
+            'End If
+        End Function
         Private Function GetTimelineGQL(ByVal Cursor As String, ByVal Token As CancellationToken) As String
             Const none_cursor$ = "none"
-            Dim nextCursor$ = String.Empty, hasNextPage$ = String.Empty
+            Dim nextCursor$ = String.Empty
+            Dim hasNextPage As Boolean = False
             Dim vars$
 
             ThrowAny(Token)
@@ -98,14 +108,18 @@ Namespace API.Instagram
             ChangeResponserMode(True)
 
             If Cursor.IsEmptyString Then
-                vars = "{""data"":{""count"":50,""include_relationship_info"":true,""latest_besties_reel_media"":true,""latest_reel_media"":true},""username"":""" &
+                vars = "{""data"":{""count"":" & PostNumberPerRequest & ",""include_relationship_info"":true,""latest_besties_reel_media"":true,""latest_reel_media"":true},""username"":""" &
                         NameTrue & """,""__relay_internal__pv__PolarisShareMenurelayprovider"":false}"
+                'vars = "{""data"":{""count"":" & PostNumberPerRequest & ",""include_reel_media_seen_timestamp"":true,""include_relationship_info"":true,""latest_besties_reel_media"":true,""latest_reel_media"":true},""username"":""" &
+                '        NameTrue & """,""__relay_internal__pv__PolarisShareMenurelayprovider"":false}"
                 vars = String.Format(GQL_URL_PATTERN_VARS, GQL_Timeline_DocId, Token_lsd, Token_dtsg_Var, GQL_Timeline_FbFriendlyName,
                                      SymbolsConverter.ASCII.EncodeSymbolsOnly(vars))
                 UpdateHeadersGQL(GQL_Timeline_FbFriendlyName)
             Else
-                vars = "{""after"":""" & Cursor & """,""before"":null,""data"":{""count"":50,""include_relationship_info"":true,""latest_besties_reel_media"":true,""latest_reel_media"":true},""first"":50,""last"":null,""username"":""" &
+                vars = "{""after"":""" & Cursor & """,""before"":null,""data"":{""count"":" & PostNumberPerRequest & ",""include_relationship_info"":true,""latest_besties_reel_media"":true,""latest_reel_media"":true},""first"":" & PostNumberPerRequest & ",""last"":null,""username"":""" &
                         NameTrue & """,""__relay_internal__pv__PolarisShareMenurelayprovider"":false}"
+                'vars = "{""after"":""" & Cursor & """,""before"":null,""data"":{""count"":" & PostNumberPerRequest & ",""include_reel_media_seen_timestamp"":true,""include_relationship_info"":true,""latest_besties_reel_media"":true,""latest_reel_media"":true},""first"":" & PostNumberPerRequest & ",""last"":null,""username"":""" &
+                '        NameTrue & """}"
                 vars = String.Format(GQL_URL_PATTERN_VARS, GQL_Timeline_DocId_Second, Token_lsd, Token_dtsg_Var, GQL_Timeline_FbFriendlyName_Second,
                                      SymbolsConverter.ASCII.EncodeSymbolsOnly(vars))
                 UpdateHeadersGQL(GQL_Timeline_FbFriendlyName_Second)
@@ -140,7 +154,8 @@ Namespace API.Instagram
         End Function
         Private Function GetHighlightsGQL_List() As List(Of String)
 
-            Dim nextCursor$ = String.Empty, hasNextPage$ = String.Empty
+            Dim nextCursor$ = String.Empty
+            Dim hasNextPage As Boolean = False
             Dim i% = -1
             Dim hList As New List(Of String)
             Dim tmpList As New List(Of String)
@@ -178,7 +193,9 @@ Namespace API.Instagram
             Dim tmpList As New List(Of String)
             Dim i% = -1
             If StoriesList.ListExists Then
-                tmpList.AddRange(StoriesList.Take(10))
+                'TODO: 5 Instagram stories
+                'tmpList.AddRange(StoriesList.Take(10))
+                tmpList.AddRange(StoriesList.Take(5))
                 StoriesList.RemoveRange(0, tmpList.Count)
 
                 Dim vars$ = String.Format(GQL_URL_PATTERN_VARS, GQL_Highlights_DocId_Second, Token_lsd, Token_dtsg_Var, GQL_Highlights_FbFriendlyName_Second,
@@ -238,11 +255,9 @@ Namespace API.Instagram
         Private Function GetReelsGQL(ByVal Cursor As String) As String
             GetReelsGQL_SetEnvir = True
 
-            Dim errData$ = String.Empty
-            If Cursor.IsEmptyString And Not ValidateBaseTokens() Then GetPageTokens()
-            If Cursor.IsEmptyString And Not ValidateBaseTokens(errData) Then ValidateBaseTokens_Error(errData)
+            UpdateTokens(Cursor.IsEmptyString)
 
-            Dim vars$ = """data"":{""include_feed_video"":true,""page_size"":50,""target_user_id"":""" & ID & """}"
+            Dim vars$ = """data"":{""include_feed_video"":true,""page_size"":" & PostNumberPerRequest & ",""target_user_id"":""" & ID & """}"
             If Not Cursor.IsEmptyString Then vars = $"""after"":""{Cursor}"",""before"":null,{vars},""first"":4,""last"":null"
             vars = String.Format(GQL_URL_PATTERN_VARS, GQL_Reels_DocId, Token_lsd, Token_dtsg_Var, GQL_Reels_FbFriendlyName,
                                  SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & vars & "}"))
@@ -258,10 +273,10 @@ Namespace API.Instagram
             Dim vars$
             If Cursor.IsEmptyString Then
                 vars = String.Format(GQL_URL_PATTERN_VARS, GQL_Tagged_DocId, Token_lsd, Token_dtsg_Var, GQL_Tagged_FbFriendlyName,
-                                     SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & $"""count"":50,""user_id"":""{ID}""" & "}"))
+                                     SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & $"""count"":{PostNumberPerRequest},""user_id"":""{ID}""" & "}"))
             Else
                 vars = String.Format(GQL_URL_PATTERN_VARS, GQL_Tagged_DocId, Token_lsd, Token_dtsg_Var, GQL_Tagged_FbFriendlyName,
-                                     SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & $"""after"":""{Cursor}"",""before"":null,""count"":50,""first"":50,""last"":null,""user_id"":""{ID}""" & "}"))
+                                     SymbolsConverter.ASCII.EncodeSymbolsOnly("{" & $"""after"":""{Cursor}"",""before"":null,""count"":{PostNumberPerRequest},""first"":{PostNumberPerRequest},""last"":null,""user_id"":""{ID}""" & "}"))
             End If
             UpdateRequestNumber()
             ChangeResponserMode(True)
@@ -270,6 +285,13 @@ Namespace API.Instagram
         End Function
 #End Region
 #Region "ValidateBaseTokens"
+        Private Sub UpdateTokens(ByVal process As Boolean)
+            If process Then
+                Dim TokensErrData$ = String.Empty
+                If Not ValidateBaseTokens() Then GetPageTokens()
+                If Not ValidateBaseTokens(TokensErrData) Then ValidateBaseTokens_Error(TokensErrData)
+            End If
+        End Sub
         Protected Overridable Overloads Function ValidateBaseTokens() As Boolean
             Return ValidateBaseTokens(Nothing)
         End Function
@@ -307,6 +329,10 @@ Namespace API.Instagram
             Try
                 If Not r.IsEmptyString Then
                     ResetBaseTokens()
+                    If ID.IsEmptyString Then
+                        Dim __id$ = RegexReplace(r, Regex_ProfileID)
+                        If CLng(AConvert(Of Long)(__id, 0, EDP.ReturnValue)) <> 0 Then ID = __id
+                    End If
                     Select Case Attempt
                         Case 0
                             Dim rr As RParams = RParams.DM(PageTokenRegexPatternDefault, 0, RegexReturn.List, EDP.ReturnValue)

SCrawler/API/Instagram/UserData.vb

@@ -39,6 +39,8 @@ Namespace API.Instagram
         Private Const Name_TaggedChecked As String = "TaggedChecked"
         Private Const Name_ForceUpdateUserName As String = "ForceUpdateUserName"
         Private Const Name_ForceUpdateUserInfo As String = "ForceUpdateUserInfo"
+        Private Const Name_IsVerifiedProfile As String = "IsVerifiedProfile"
+        Private Const Name_IsVerifiedProfile_Checked As String = "IsVerifiedProfile_Checked"
 #End Region
 #Region "Declarations"
         Friend Structure PostKV : Implements IEContainerProvider
@@ -115,6 +117,13 @@ Namespace API.Instagram
         Private UserNameRequested As Boolean = False
         Friend Property ForceUpdateUserName As Boolean = False
         Friend Property ForceUpdateUserInfo As Boolean = False
+        Friend Property IsVerifiedProfile As Boolean = False
+        Friend Property IsVerifiedProfile_Checked As Boolean = False
+        Private ReadOnly Property PostNumberPerRequest As Integer
+            Get
+                With MySiteSettings : Return If(IsVerifiedProfile, .PostNumberVerified, .PostNumberVerifiedNot).Value : End With
+            End Get
+        End Property
 #End Region
 #Region "Loader"
         Protected Overrides Sub LoadUserInformation_OptionalFields(ByRef Container As XmlFile, ByVal Loading As Boolean)
@@ -136,6 +145,8 @@ Namespace API.Instagram
                     TaggedChecked = .Value(Name_TaggedChecked).FromXML(Of Boolean)(False)
                     ForceUpdateUserName = .Value(Name_ForceUpdateUserName).FromXML(Of Boolean)(False)
                     ForceUpdateUserInfo = .Value(Name_ForceUpdateUserInfo).FromXML(Of Boolean)(False)
+                    IsVerifiedProfile = .Value(Name_IsVerifiedProfile).FromXML(Of Boolean)(False)
+                    IsVerifiedProfile_Checked = .Value(Name_IsVerifiedProfile_Checked).FromXML(Of Boolean)(False)
                 Else
                     .Add(Name_LastCursor, LastCursor)
                     .Add(Name_FirstLoadingDone, FirstLoadingDone.BoolToInteger)
@@ -153,6 +164,8 @@ Namespace API.Instagram
                     .Add(Name_TaggedChecked, TaggedChecked.BoolToInteger)
                     .Add(Name_ForceUpdateUserName, ForceUpdateUserName.BoolToInteger)
                     .Add(Name_ForceUpdateUserInfo, ForceUpdateUserInfo.BoolToInteger)
+                    .Add(Name_IsVerifiedProfile, IsVerifiedProfile.BoolToInteger)
+                    .Add(Name_IsVerifiedProfile_Checked, IsVerifiedProfile_Checked.BoolToInteger)
                 End If
             End With
         End Sub
@@ -179,6 +192,9 @@ Namespace API.Instagram
 
                     PutImageVideoFolder = .PutImageVideoFolder
 
+                    IsVerifiedProfile = .IsVerifiedProfile
+                    If IsVerifiedProfile Then IsVerifiedProfile_Checked = True
+
                     ForceUpdateUserName = .ForceUpdateUserName
                     ForceUpdateUserInfo = .ForceUpdateUserInfo
                 End With
@@ -663,6 +679,7 @@ Namespace API.Instagram
             Dim StoriesList As List(Of String) = Nothing
             Dim StoriesRequested As Boolean = False
             Dim dValue% = 1
+            Dim __idIsEmpty As Boolean = ID.IsEmptyString
             LastCursor = Cursor
             Try
                 Do While dValue = 1
@@ -676,7 +693,6 @@ Namespace API.Instagram
                         Dim HasNextPage As Boolean = False
                         Dim EndCursor$ = String.Empty
                         Dim PostID$ = String.Empty, PostDate$ = String.Empty, SpecFolder$ = String.Empty
-                        Dim TokensErrData$ = String.Empty
                         Dim PostIDKV As PostKV
                         Dim ENode() As Object = Nothing
                         Dim processGetResponse As Boolean = True
@@ -684,14 +700,11 @@ Namespace API.Instagram
 
                         'Check environment
                         If Not IsSavedPosts Then
-                            If ID.IsEmptyString Then GetUserData()
-                            If ID.IsEmptyString Then UserExists = False : _ForceSaveUserInfoOnException = True : Throw New Plugin.ExitException("can't get user ID")
-                            If _UseGQL And Cursor.IsEmptyString And Not Section = Sections.SavedPosts Then
-                                If Not ValidateBaseTokens() Then GetPageTokens()
-                                If Not ValidateBaseTokens(TokensErrData) Then ValidateBaseTokens_Error(TokensErrData)
-                            End If
+                            If _UseGQL And Cursor.IsEmptyString And Not Section = Sections.SavedPosts Then UpdateTokens(True)
+                            If ID.IsEmptyString Or __idIsEmpty Or Not IsVerifiedProfile_Checked Then GetUserData(Token)
+                            If ID.IsEmptyString Then UserExists = False : _ForceSaveUserInfo = True : Throw New Plugin.ExitException("can't get user ID")
                             If ForceUpdateUserName Then GetUserNameById()
-                            If ForceUpdateUserInfo Then GetUserData()
+                            If ForceUpdateUserInfo Then GetUserData(Token)
                         End If
 
                         'Create query
@@ -703,7 +716,7 @@ Namespace API.Instagram
                                     MySiteSettings.TooManyRequests(False)
                                     GoTo NextPageBlock
                                 Else
-                                    URL = $"https://www.instagram.com/api/v1/feed/user/{NameTrue}/username/?count=50" &
+                                    URL = $"https://www.instagram.com/api/v1/feed/user/{NameTrue}/username/?count={PostNumberPerRequest}" &
                                            If(Cursor.IsEmptyString, String.Empty, $"&max_id={Cursor}")
                                     ENode = Nothing
                                 End If
@@ -726,7 +739,7 @@ Namespace API.Instagram
                                     ENode = {"data", "xdt_api__v1__usertags__user_id__feed_connection"}
                                     processGetResponse = False
                                 Else
-                                    Dim vars$ = "{""id"":" & ID & ",""first"":50,""after"":""" & Cursor & """}"
+                                    Dim vars$ = "{""id"":" & ID & $",""first"":{PostNumberPerRequest},""after"":""" & Cursor & """}"
                                     vars = SymbolsConverter.ASCII.EncodeSymbolsOnly(vars)
                                     URL = $"https://www.instagram.com/graphql/query/?doc_id=17946422347485809&variables={vars}"
                                     ENode = {"data", "user", "edge_user_to_photos_of_you"}
@@ -1241,25 +1254,37 @@ NextPageBlock:
         End Sub
 #End Region
 #Region "GetUserId, GetUserName"
-        Private Sub GetUserData()
+        Private Sub GetUserData(ByVal Token As CancellationToken)
             Dim __idFound As Boolean = False
             If ForceUpdateUserInfo Then ForceUpdateUserInfo = False : _ForceSaveUserInfo = True
             Try
-                ChangeResponserMode(False)
-                UpdateRequestNumber()
-                Dim r$ = Responser.GetResponse($"https://i.instagram.com/api/v1/users/web_profile_info/?username={NameTrue}")
+                Dim r$
+                Dim ____dataGql As Boolean = _UseGQL Or CBool(MySiteSettings.USE_GQL_UserData.Value)
+                If ____dataGql Then
+                    UpdateTokens(True)
+                    r = GetUserDataGQL(Token)
+                    If Not _UseGQL Then ChangeResponserMode(_UseGQL)
+                Else
+                    ChangeResponserMode(False)
+                    UpdateRequestNumber()
+                    r = Responser.GetResponse($"https://i.instagram.com/api/v1/users/web_profile_info/?username={NameTrue}")
+                End If
+
                 If Not r.IsEmptyString Then
                     Using j As EContainer = JsonDocument.Parse(r)
                         If Not j Is Nothing AndAlso j.Contains({"data", "user"}) Then
                             With j({"data", "user"})
-                                ID = .Value("id")
+                                If Not ____dataGql Or ID.IsEmptyString Then ID = .Value("id")
                                 __idFound = True
                                 UserSiteNameUpdate(.Value("full_name"))
+                                IsVerifiedProfile = .Value("is_verified").FromXML(Of Boolean)(False)
+                                IsVerifiedProfile_Checked = True
                                 Dim descr$ = .Value("biography")
                                 If If(.Item("bio_links")?.Count, 0) > 0 Then descr.StringAppend(.Item("bio_links").Select(Function(bl) bl.Value("url")).ListToString(vbNewLine), vbNewLine)
                                 Dim eUrl$ = .Value("external_url")
                                 If Not eUrl.IsEmptyString AndAlso (descr.IsEmptyString OrElse Not descr.Contains(eUrl)) Then descr.StringAppendLine(eUrl)
                                 UserDescriptionUpdate(descr)
+
                                 Dim f As New SFile With {.Path = DownloadContentDefault_GetRootDir(), .Name = "ProfilePicture", .Extension = "jpg"}
                                 f = SFile.IndexReindex(f)
                                 If Not f.Exists Then

SCrawler/API/PornHub/M3U8.vb

@@ -22,7 +22,7 @@ Namespace API.PornHub
             If Not r.IsEmptyString Then
                 Dim files As List(Of Sizes) = RegexFields(Of Sizes)(r, {Regex_M3U8_FilesList}, {1, 2}, EDP.ReturnValue)
                 Dim file$
-                If files.ListExists Then files.RemoveAll(Function(f) f.Value = 0 Or (Not DownloadUHD And f.Value > 1080))
+                If files.ListExists Then files.RemoveAll(Function(f) f.Value = 0 Or (Not DownloadUHD And f.Value > 1080) Or (f.Data.IsEmptyString OrElse f.Data.Contains("""")))
                 If files.ListExists Then
                     files.Sort()
                     file = files(0).Data

SCrawler/API/PornHub/UserData.vb

@@ -258,7 +258,6 @@ Namespace API.PornHub
         Private _PageVideosRepeat As Integer = 0
         Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
             Try
-                UpdateM3U8URLS = False
                 PlaylistToken = String.Empty
                 Responser.ResetStatus()
                 _PageVideosRepeat = 0
@@ -793,31 +792,29 @@ Namespace API.PornHub
         End Sub
 #End Region
 #Region "Download content"
-        Private UpdateM3U8URLS As Boolean = False
-        Private UpdateM3U8URLS_Error As Boolean = False
         Protected Overrides Sub DownloadContent(ByVal Token As CancellationToken)
-            Try : DownloadContentDefault(Token) : Finally : UpdateM3U8URLS = False : End Try
+            DownloadContentDefault(Token)
         End Sub
         Protected Overloads Overrides Function DownloadM3U8(ByVal URL As String, ByVal Media As UserMedia, ByVal DestinationFile As SFile,
                                                             ByVal Token As CancellationToken) As SFile
-            UpdateM3U8URLS_Error = False
-            Return DownloadM3U8(URL, Media, DestinationFile, Token, UpdateM3U8URLS)
+            Return DownloadM3U8(URL, Media, DestinationFile, Token, 0)
         End Function
         Private Overloads Function DownloadM3U8(ByVal URL As String, ByVal Media As UserMedia, ByVal DestinationFile As SFile,
-                                                ByVal Token As CancellationToken, ByVal Second As Boolean) As SFile
+                                                ByVal Token As CancellationToken, ByVal Round As Integer) As SFile
+            Const MaxRound% = 2
             Try
-                If Second Then
+                If Round > 0 Then
                     Dim r$ = Responser.Curl(Media.URL_BASE,, EDP.ReturnValue)
                     If Not r.IsEmptyString Then Media.URL = CreateVideoURL(r).IfNullOrEmpty(URL) : URL = Media.URL
                 End If
                 Dim f As SFile = M3U8.Download(URL, Responser, DestinationFile, DownloadUHD, Token, Progress, Not IsSingleObjectDownload)
-                If Not f.Exists And Not Second Then UpdateM3U8URLS = True : f = DownloadM3U8(URL, Media, DestinationFile, Token, True)
+                If Not f.Exists Then f = Nothing
+                If f.IsEmptyString And Round < MaxRound Then f = DownloadM3U8(URL, Media, DestinationFile, Token, Round + 1)
                 Return f
             Catch ex As Exception
-                If Not UpdateM3U8URLS_Error Then
-                    UpdateM3U8URLS_Error = True
+                If Round < MaxRound Then
                     Thread.Sleep(1000)
-                    Return DownloadM3U8(URL, Media, DestinationFile, Token, True)
+                    Return DownloadM3U8(URL, Media, DestinationFile, Token, Round + 1)
                 End If
                 Return Nothing
             End Try
@@ -917,7 +914,6 @@ Namespace API.PornHub
 #End Region
 #Region "DownloadSingleObject"
         Protected Overrides Sub DownloadSingleObject_GetPosts(ByVal Data As IYouTubeMediaContainer, ByVal Token As CancellationToken)
-            UpdateM3U8URLS = False
             _TempMediaList.Add(New UserMedia(Data.URL, UTypes.VideoPre))
             ReparseVideo(Token, True, Data)
         End Sub

SCrawler/API/PornHub/UserExchangeOptions.vb

@@ -44,7 +44,7 @@ Namespace API.PornHub
             DownloadGifs = Not v = CheckState.Unchecked
             MySettings = s
         End Sub
-        Friend Overrides Sub Apply(ByRef u As IPSite)
+        Friend Overrides Sub Apply(ByRef u As UserDataBase)
             MyBase.Apply(u)
             With DirectCast(u, UserData)
                 .DownloadUHD = DownloadUHD

SCrawler/API/ThisVid/UserExchangeOptions.vb

@@ -35,7 +35,7 @@ Namespace API.ThisVid
             DifferentFolders = u.DifferentFolders
             MySettings = u.HOST.Source
         End Sub
-        Friend Overrides Sub Apply(ByRef u As IPSite)
+        Friend Overrides Sub Apply(ByRef u As UserDataBase)
             MyBase.Apply(u)
             With DirectCast(u, UserData)
                 .DownloadPublic = DownloadPublic

SCrawler/API/ThreadsNet/UserData.vb

@@ -184,7 +184,6 @@ Namespace API.ThreadsNet
             If uex.UserNotFound Then
                 UserExists = False
                 _ForceSaveUserInfo = True
-                _ForceSaveUserInfoOnException = True
             ElseIf ThrowEx Then
                 Throw New ExitException(uex.ErrMessage) With {.SimpleLogLine = True}
             Else
@@ -404,7 +403,6 @@ Namespace API.ThreadsNet
                     If _IdChanged Then
                         If Not idStr.IsEmptyString Then UserDescriptionUpdate(idStr, True, True, True)
                         _ForceSaveUserInfo = True
-                        _ForceSaveUserInfoOnException = True
                     End If
                 End If
                 Return Valid

SCrawler/API/TikTok/UserData.vb

@@ -192,10 +192,14 @@ Namespace API.TikTok
             UserCache.DisposeIfReady(False)
             UserCache = Nothing
         End Sub
+        Private StoryStatus As Boolean = True
+        Private StoryStatusFound As Boolean = False
         Protected Overloads Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
             ValidateCache()
+            StoryStatus = True
+            StoryStatusFound = False
             If GetTimeline Then DownloadDataF(Sections.Timeline, Token)
-            If GetStoriesUser Then DownloadDataF(Sections.UserStories, Token)
+            If GetStoriesUser And StoryStatus Then DownloadDataF(Sections.UserStories, Token)
             If GetReposts Then DownloadDataF(Sections.Reposts, Token)
         End Sub
         Protected Overloads Sub DownloadDataF(ByVal Section As Sections, ByVal Token As CancellationToken)
@@ -271,8 +275,8 @@ Namespace API.TikTok
                             Else
                                 .TempPostsList = New List(Of String)
                             End If
-                            .Execute(CreateGDLCommand(URL, gdlCmd))
-                            If Not PhotosDownloaded Then _ForceSaveUserInfo = True : _ForceSaveUserInfoOnException = True
+                            .Execute(CreateGDLCommand(URL, gdlCmd,,,, CObj(IIf(Section = Sections.Timeline, dateAfter, Nothing))))
+                            If Not PhotosDownloaded Then _ForceSaveUserInfo = True
                             PhotosDownloaded = True
                         End With
                     End Using
@@ -436,6 +440,10 @@ Namespace API.TikTok
                                             End If
                                         End With
                                     Else
+                                        If Section = Sections.Timeline And Not StoryStatusFound And j.ContainsF({0, "webapp.user-detail", "userInfo", "user"}, "UserStoryStatus") Then
+                                            StoryStatusFound = True
+                                            StoryStatus = j.ItemF({0, "webapp.user-detail", "userInfo", "user"}, "UserStoryStatus").Value.FromXML(Of Boolean)(False)
+                                        End If
                                         With j.ItemF({0, "webapp.video-detail", "itemInfo", "itemStruct"})
                                             If .ListExists Then
                                                 postID = .Value("id")
@@ -604,13 +612,17 @@ Namespace API.TikTok
 #End Region
 #Region "GDL Support"
         Private Function CreateGDLCommand(ByVal URL As String, Optional ByVal SectionCommand As String = Nothing,
-                                          Optional ByVal IsDownload As Boolean = False, Optional ByVal Output As SFile = Nothing) As String
+                                          Optional ByVal IsDownload As Boolean = False, Optional ByVal Output As SFile = Nothing,
+                                          Optional ByVal DateBefore As Date? = Nothing, Optional ByVal DateAfter As Date? = Nothing) As String
             Dim command$ = $"""{Settings.GalleryDLFile}"" "
             If Not IsDownload Then
                 command &= "--verbose --no-download --no-skip --write-pages "
             Else
                 command &= $"--dest ""{Output.PathNoSeparator}"" "
             End If
+            If DateBefore.HasValue Or DateAfter.HasValue Then
+                With If(DateAfter, DateBefore).Value : command &= $"--filter ""date {IIf(DateAfter.HasValue, ">", "<")} datetime({ .Year}, { .Month}, { .Day}) or terminate()"" " : End With
+            End If
             If Not CBool(If(IsSingleObjectDownload, MySettings.UseParsedVideoDateSTD, MySettings.UseParsedVideoDate).Value) Then command &= "--no-mtime "
             command &= $"{SectionCommand} {URL}"
             Return command

SCrawler/API/Twitter/UserData.vb

@@ -297,10 +297,17 @@ Namespace API.Twitter
         Private Const DEBUG_PROFILE As Boolean = False
         Private Const DEBUG_LEAVE_CACHE As Boolean = False
         Private JsonNullErr As Boolean = False
+        Private ____UserExists As Boolean = True
+        Private NotUserExistsAttempts As Integer = 0
+        Friend Overrides Sub DownloadData(ByVal Token As CancellationToken)
+            ____UserExists = UserExists
+            MyBase.DownloadData(Token)
+        End Sub
         Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
             Try
                 GDL_REQUESTS_COUNT = 0
                 JsonNullErr = False
+                NotUserExistsAttempts = 0
                 If MySettings.LIMIT_ABORT Then
                     Throw New TwitterLimitException(Me)
                 Else
@@ -317,6 +324,14 @@ Namespace API.Twitter
                         End If
                         LikesPosts.Clear()
                         If _ContentList.Count > 0 Then _DataNames.ListAddList(_ContentList.Select(Function(c) c.File.File), LAP.ClearBeforeAdd, LAP.NotContainsOnly)
+                        If Not ____UserExists Then
+                            For i% = 0 To 1
+                                NotUserExistsAttempts += 1
+                                DownloadData_Timeline(Token)
+                                If UserExists Then ____UserExists = True : Exit For
+                            Next
+                        End If
+                        If Not UserExists Then Exit Sub
                         DownloadData_Timeline(Token)
                         If _TempMediaList.Count = 0 And LikesPosts.Count = 0 And JsonNullErr Then Throw New Plugin.ExitException("No deserialized data found")
                         LoadSavePostsKV(False)
@@ -366,6 +381,7 @@ Namespace API.Twitter
                 Dim j As EContainer, rootNode As EContainer, optionalNode As EContainer, workingNode As EContainer, tmpNode As EContainer, nn As EContainer = Nothing
                 Dim multiMode As Boolean = IsMultiMode
                 Dim currentModel As DownloadModels = DownloadModels.Undefined
+                Dim onlyUpdateUser As Boolean = Not ____UserExists
 
                 Dim __parseContainer As Func(Of EContainer, Boolean) =
                     Function(ByVal ee As EContainer) As Boolean
@@ -478,6 +494,7 @@ Namespace API.Twitter
                                 End If
                                 'parse files
                                 For i = 0 To timelineFiles.Count - 1
+                                    If userInfoParsed And onlyUpdateUser Then Exit Sub
                                     j = JsonDocument.Parse(timelineFiles(i).GetText, jsonArgs)
                                     If jsonArgs.State = WebDocumentEventArgs.States.Error Then
                                         jsonArgs.Reset(Token)
@@ -506,7 +523,7 @@ Namespace API.Twitter
                                                                     With .ItemF({0, "content", "items", 0, "item", "itemContent", "tweet_results", "result", "tweet", "community_results", "result"})
                                                                         If .ListExists Then
                                                                             If ID = .Value("id_str") Then
-                                                                                UserSiteNameUpdate(.Value("name"))
+                                                                                UserSiteNameUpdate(.Value("name"), True)
                                                                                 UserDescriptionUpdate(.Value("description"))
 
                                                                                 icon = .Value({"custom_banner_media", "media_info"}, "original_img_url").
@@ -526,12 +543,20 @@ Namespace API.Twitter
                                                         If .ListExists Then
                                                             If ID.IsEmptyString Then ID = .Value("rest_id")
                                                             icon = .Value({"avatar"}, "image_url")
-                                                            UserSiteNameUpdate(.Value({"core"}, "name"))
+                                                            UserSiteNameUpdate(.Value({"core"}, "name"), True)
                                                             Dim tScreenName$ = .Value({"core"}, "screen_name")
                                                             With .Item({"legacy"})
                                                                 If .ListExists Then
+                                                                    If onlyUpdateUser Then
+                                                                        If Not NameTrue = tScreenName Then
+                                                                            Dim uStr$ = $"username changed from '{NameTrue}' to '{tScreenName}'"
+                                                                            LogError(Nothing, uStr)
+                                                                            UserDescriptionUpdate(uStr, True, True, True)
+                                                                        End If
+                                                                        NameTrue = tScreenName
+                                                                    End If
                                                                     If .Value("screen_name").IfNullOrEmpty(tScreenName).StringToLower = NameTrue.ToLower Then
-                                                                        UserSiteNameUpdate(.Value("name"))
+                                                                        UserSiteNameUpdate(.Value("name"), True)
                                                                         UserDescriptionUpdate(.Value("description"))
 
                                                                         If icon.IsEmptyString Then icon = .Value("profile_image_url_https")
@@ -846,6 +871,23 @@ nextpIndx:
         Private Class TwitterGDL : Inherits GDL.GDLBatch
             Private ReadOnly KillOnLimit As Boolean
             Friend LimitReached As Boolean = False
+            Private _GetOnlyUserInfo As Boolean = False
+            Friend Overrides Property MyWorkingDirectory As SFile
+                Get
+                    Return If(MyBase.MyWorkingDirectory.IsEmptyString, If(FileExchanger?.RootDirectory, MyBase.MyWorkingDirectory), MyBase.MyWorkingDirectory)
+                End Get
+                Set(ByVal dir As SFile)
+                    MyBase.MyWorkingDirectory = dir
+                End Set
+            End Property
+            Friend Property GetOnlyUserInfo As Boolean
+                Get
+                    Return _GetOnlyUserInfo And Not MyWorkingDirectory.IsEmptyString
+                End Get
+                Set(ByVal __GetOnlyUserInfo As Boolean)
+                    _GetOnlyUserInfo = __GetOnlyUserInfo
+                End Set
+            End Property
             Friend Sub New(ByVal Dir As SFile, ByVal _Token As CancellationToken, ByVal _KillOnLimit As Boolean)
                 MyBase.New(_Token,, Dir)
                 KillOnLimit = _KillOnLimit
@@ -855,10 +897,14 @@ nextpIndx:
             End Function
             Private Function IdExists(ByVal Value As String) As Boolean
                 Try
-                    Value = Value.StringTrim
-                    If Not Value.IsEmptyString AndAlso (Value.StartsWith("*") Or Value.StartsWith(".\gallery-dl\")) Then
-                        Dim id$ = Value.Split("\").Last.Split(".").First.Split("_").First
-                        If Not id.IsEmptyString Then Return TempPostsList.Contains(id)
+                    If GetOnlyUserInfo Then
+                        Return CheckForData()
+                    Else
+                        Value = Value.StringTrim
+                        If Not Value.IsEmptyString AndAlso (Value.StartsWith("*") Or Value.StartsWith(".\gallery-dl\")) Then
+                            Dim id$ = Value.Split("\").Last.Split(".").First.Split("_").First
+                            If Not id.IsEmptyString Then Return TempPostsList.Contains(id)
+                        End If
                     End If
                 Catch ex As Exception
                 End Try
@@ -867,8 +913,14 @@ nextpIndx:
             Protected Overrides Async Sub ErrorDataReceiver(ByVal Sender As Object, ByVal e As DataReceivedEventArgs)
                 Await Task.Run(Sub() CheckForLimit(e.Data))
             End Sub
+            Private Function CheckForData()
+                If GetOnlyUserInfo Then
+                    If SFile.GetFiles(MyWorkingDirectory, "*.txt",, EDP.ReturnValue).Count > 2 Then Return True
+                End If
+                Return False
+            End Function
             Private Sub CheckForLimit(ByVal Value As String)
-                If Token.IsCancellationRequested Or (KillOnLimit AndAlso Not ProcessKilled AndAlso
+                If CheckForData() Or Token.IsCancellationRequested Or (KillOnLimit AndAlso Not ProcessKilled AndAlso
                    Not Value.IsEmptyString AndAlso (Value.ToLower.Contains("for rate limit reset") OrElse
                                                     Not CStr(RegexReplace(Value, GdlLimitRegEx)).IsEmptyString)) Then
                     LimitReached = True
@@ -1017,11 +1069,12 @@ nextpIndx:
                     .AutoClear = True,
                     .AutoReset = True,
                     .CommandPermanent = $"chcp {BatchExecutor.UnicodeEncoding}",
-                    .FileExchanger = confCache
+                    .FileExchanger = confCache,
+                    .GetOnlyUserInfo = NotUserExistsAttempts > 0
                 }
                     tgdl.FileExchanger.DeleteCacheOnDispose = False
                     tgdl.FileExchanger.DeleteRootOnDispose = False
-                    For i As Byte = 0 To IIf(IsCommunity, 0, 3)
+                    For i As Byte = 0 To IIf(IsCommunity Or NotUserExistsAttempts > 0, 0, 3)
                         dir = rootDir.NewPath
                         dir.Exists(SFO.Path, True, EDP.ThrowException)
                         outList.Add(dir)
@@ -1032,13 +1085,28 @@ nextpIndx:
                         Else
                             command &= GdlGetIdFilterString()
                         End If
-                        Select Case i
-                            Case 0 : command &= $"{urlPrePattern}{NameTrue}/media" : currentModel = DownloadModels.Media : process = dm.Contains(currentModel) Or IsCommunity
-                            Case 1 : command &= $"{urlPrePattern}{NameTrue}" : currentModel = DownloadModels.Profile : process = dm.Contains(currentModel)
-                            Case 2 : command &= $"-o search-endpoint=graphql https://x.com/search?q=from:{NameTrue}+include:nativeretweets" : currentModel = DownloadModels.Search : process = dm.Contains(currentModel) And Not IsCommunity
-                            Case 3 : command &= $"{urlPrePattern}{NameTrue}/likes" : currentModel = DownloadModels.Likes : process = dm.Contains(currentModel)
-                            Case Else : process = False
-                        End Select
+                        If NotUserExistsAttempts > 0 Then
+                            Select Case NotUserExistsAttempts
+                                Case 1 : command &= $"{urlPrePattern}{NameTrue}/media" : currentModel = DownloadModels.Media : process = True
+                                Case 2
+                                    If ID.IsEmptyString Then
+                                        process = False
+                                    Else
+                                        command &= $"https://twitter.com/intent/user?user_id={ID}"
+                                        currentModel = DownloadModels.Media
+                                        process = True
+                                    End If
+                                Case Else : process = False
+                            End Select
+                        Else
+                            Select Case i
+                                Case 0 : command &= $"{urlPrePattern}{NameTrue}/media" : currentModel = DownloadModels.Media : process = dm.Contains(currentModel) Or IsCommunity
+                                Case 1 : command &= $"{urlPrePattern}{NameTrue}" : currentModel = DownloadModels.Profile : process = dm.Contains(currentModel)
+                                Case 2 : command &= $"-o search-endpoint=graphql https://x.com/search?q=from:{NameTrue}+include:nativeretweets" : currentModel = DownloadModels.Search : process = dm.Contains(currentModel) And Not IsCommunity
+                                Case 3 : command &= $"{urlPrePattern}{NameTrue}/likes" : currentModel = DownloadModels.Likes : process = dm.Contains(currentModel)
+                                Case Else : process = False
+                            End Select
+                        End If
                         '#If DEBUG Then
                         'Debug.WriteLine(command)
                         '#End If
@@ -1122,10 +1190,12 @@ nextpIndx:
             Dim cache As CacheKeeper = Nothing
             Try
                 If ContentMissingExists Or (_ReparseLikes And LikesPosts.Count > 0) Then
+                    Const __entries$ = "entries"
                     Dim m As UserMedia, mTmp As UserMedia
                     Dim PostDate$
                     Dim nodes As List(Of String()) = GetContainerSubnodes()
                     Dim node$()
+                    Dim entriesNode As Predicate(Of EContainer) = Function(ee) ee.Contains(__entries)
                     Dim j As EContainer, n As EContainer
                     Dim f As SFile
                     Dim i%, ii%
@@ -1161,7 +1231,7 @@ nextpIndx:
                                             f = GDLRenameFile(files(ii), ii)
                                             j = JsonDocument.Parse(f.GetText)
                                             If Not j Is Nothing Then
-                                                With j.ItemF({"data", 0, "instructions", 0, "entries"})
+                                                With j.ItemF({"data", 0, "instructions", entriesNode, __entries})
                                                     If .ListExists Then
                                                         If IsSingleObjectDownload Or DownloadBroadcasts Then
                                                             mTmp = ExtractBroadcast(.Self, m.Post.ID, String.Empty, nodes)

SCrawler/API/Xhamster/UserData.vb

@@ -298,6 +298,7 @@ Namespace API.Xhamster
                         containerNodes.Add({"videoListComponent", "videoThumbProps"})
                     Else
                         containerNodes.Add({"userGalleriesCollection"})
+                        containerNodes.Add({"contentComponent", "items"})
                     End If
                 End If
 
@@ -744,12 +745,13 @@ Namespace API.Xhamster
                 If IsSingleObjectDownload Then
                     cc = Settings.Cache
                 Else
-                    If MyCache Is Nothing Then MyCache = CreateCache() : MyCache.Validate()
+                    If MyCache Is Nothing Then MyCache = CreateCache()
                     cc = MyCache
                 End If
+                cc.Validate()
                 Dim path As SFile = cc.NewPath
                 Dim c$ = If(MySettings.CookiesNetscapeFile.Exists, $" --no-cookies-from-browser --cookies ""{MySettings.CookiesNetscapeFile}""", String.Empty)
-                Dim cmd$ = $"{Settings.YtdlpFile} --write-info-json --skip-download{c} {URL} -o ""{path.PathWithSeparator}file"""
+                Dim cmd$ = $"""{Settings.YtdlpFile}"" --write-info-json --skip-download{c} {URL} -o ""{path.PathWithSeparator}file"""
                 path.Exists()
                 Using ytdlp As New YTDLP.YTDLPBatch(TokenPersonal,, path) : ytdlp.Encoding = Settings.CMDEncoding : ytdlp.Execute(cmd) : End Using
                 Return SFile.GetFiles(path, "*.json",, EDP.ReturnValue).FirstOrDefault
@@ -794,10 +796,13 @@ Namespace API.Xhamster
         Private Function YTDLPDownload(ByVal Media As UserMedia, ByVal DestinationFile As SFile, ByVal Token As CancellationToken) As SFile
             DestinationFile.Extension = "mp4"
             Dim c$ = If(MySettings.CookiesNetscapeFile.Exists, $" --no-cookies-from-browser --cookies ""{MySettings.CookiesNetscapeFile}""", String.Empty)
-            Dim cmd$ = $"{Settings.YtdlpFile} --format {DirectCast(Media.Object, XMMediaInfo).FormatID}{c} {Media.URL_BASE} -o ""{DestinationFile}"""
+            Dim cmd$ = $"""{Settings.YtdlpFile}"" --format {DirectCast(Media.Object, XMMediaInfo).FormatID}{c} {Media.URL_BASE} -o ""{DestinationFile}"""
             Using ytdlp As New YTDLP.YTDLPBatch(TokenPersonal,, DestinationFile) : ytdlp.Encoding = Settings.CMDEncoding : ytdlp.Execute(cmd) : End Using
             Return DestinationFile
         End Function
+        Protected Overrides Function DownloadContentDefault_ConvertWebp(ByVal m As UserMedia, ByVal Process As Boolean) As SFile
+            Return DownloadContentDefault_ConvertWebp_TestImg(m, Process)
+        End Function
 #End Region
 #Region "Create media"
         Private Function ExtractMedia(ByVal j As EContainer, ByVal t As UTypes, Optional ByVal UrlNode As String = "pageURL",

SCrawler/API/Xhamster/UserExchangeOptions.vb

@@ -23,7 +23,7 @@ Namespace API.Xhamster
             MyBase.New(s)
             GetMoments = s.GetMoments.Value
         End Sub
-        Friend Overrides Sub Apply(ByRef u As IPSite)
+        Friend Overrides Sub Apply(ByRef u As UserDataBase)
             MyBase.Apply(u)
             DirectCast(u, UserData).GetMoments = GetMoments
         End Sub

SCrawler/Download/Feed/DownloadFeedForm.vb

@@ -99,42 +99,50 @@ Namespace DownloadObjects
         End Sub
         Private Sub FeedRemoveCheckedMedia(ByVal MediaList As IEnumerable(Of UserMediaD), Optional ByVal OverriddenNames As List(Of String) = Nothing,
                                            Optional ByVal RemoveChecked As Boolean = True, Optional ByVal ExcludingNames As IEnumerable(Of String) = Nothing,
-                                           Optional ByVal RemoveFromDataListOnly As Boolean = False)
+                                           Optional ByVal IsAddAndRemove As Boolean = False)
             Try
-                If FeedMode = FeedModes.Special Then
-                    If LoadedFeedNames.Count > 0 Then
-                        Dim dataRemoved As Boolean = False
-                        If OverriddenNames.ListExists And Not LoadedFeedNames.ListContains(OverriddenNames) Then Exit Sub
-                        If Not RemoveFromDataListOnly Then
-                            Dim eNames As IEnumerable(Of String) = If(ExcludingNames, New String() {})
-                            With If(OverriddenNames, LoadedFeedNames)
-                                .ForEach(Sub(ByVal feedName As String)
-                                             If Not eNames.Contains(feedName) Then
-                                                 Dim indx% = Settings.Feeds.IndexOf(feedName)
-                                                 If indx >= 0 Then
-                                                     If Settings.Feeds(indx).Remove(MediaList) > 0 Then dataRemoved = True
-                                                 End If
-                                             End If
-                                         End Sub)
-                            End With
-                        End If
-                        If RemoveFromDataListOnly Then
+                If FeedMode = FeedModes.Saved Then Exit Sub
+
+                Dim dataRemoved As Boolean = False
+                If FeedMode = FeedModes.Special And OverriddenNames.ListExists And Not LoadedFeedNames.ListContains(OverriddenNames) Then Exit Sub
+                Dim eNames As IEnumerable(Of String) = If(ExcludingNames, New String() {})
+                With If(OverriddenNames, LoadedFeedNames)
+                    If FeedMode = FeedModes.Special And .ListExists Then
+                        .ForEach(Sub(ByVal feedName As String)
+                                     If Not eNames.Contains(feedName) Then
+                                         Dim indx% = Settings.Feeds.IndexOf(feedName)
+                                         If indx >= 0 Then
+                                             If Settings.Feeds(indx).Remove(MediaList) > 0 Then dataRemoved = True
+                                         End If
+                                     End If
+                                 End Sub)
+                    ElseIf FeedMode = FeedModes.Current And Not OverriddenNames.ListExists And IsAddAndRemove Then
+                        dataRemoved = Downloader.Files.ListDisposeRemove(MediaList) > 0
+                        'Downloader.FilesSave()
+                    Else
+                        Exit Sub
+                    End If
+                End With
+
+                If dataRemoved Then DataList.ListDisposeRemove(MediaList)
+
+                Select Case FeedMode
+                    Case FeedModes.Special
+                        If RemoveChecked And IsAddAndRemove Then
+                            If RemoveCheckedMedia(False) Then RefillAfterDelete()
+                        Else
                             RefillSpecialFeedsData()
-                        ElseIf dataRemoved Then
-                            DataList.ListDisposeRemove(MediaList)
+                        End If
+                    Case FeedModes.Current
+                        If dataRemoved Then
                             If RemoveChecked Then
-                                If RemoveCheckedMedia(False) Then RefillAfterDelete()
+                                RemoveCheckedMedia(False)
+                                RefillAfterDelete()
                             Else
-                                RefillSpecialFeedsData()
+                                RefillList()
                             End If
                         End If
-                    End If
-                ElseIf FeedMode = FeedModes.Current Then
-                    If OverriddenNames Is Nothing AndAlso Downloader.Files.ListDisposeRemove(MediaList) > 0 AndAlso RemoveCheckedMedia(False) Then
-                        DataList.ListDisposeRemove(MediaList)
-                        RefillAfterDelete()
-                    End If
-                End If
+                End Select
             Catch ex As Exception
                 ErrorsDescriber.Execute(EDP.SendToLog, ex, "[DownloadFeedForm.FeedRemoveCheckedMedia]")
             End Try
@@ -333,7 +341,7 @@ Namespace DownloadObjects
                 Dim c As IEnumerable(Of UserMediaD) = GetCheckedMedia()
                 If c.ListExists Then
                     f.Add(c)
-                    FeedRemoveCheckedMedia(c,,, {f.Name})
+                    FeedRemoveCheckedMedia(c,,, {f.Name}, True)
                 End If
             End If
         End Sub
@@ -352,7 +360,7 @@ Namespace DownloadObjects
                 Dim m As IEnumerable(Of UserMediaD) = GetCheckedMedia()
                 If m.ListExists Then
                     f.Remove(m)
-                    FeedRemoveCheckedMedia(m, {f.Name}.ToList)
+                    FeedRemoveCheckedMedia(m, {f.Name}.ToList,,, False)
                 End If
             End If
         End Sub
@@ -1000,14 +1008,14 @@ Namespace DownloadObjects
             Dim m As IEnumerable(Of UserMediaD) = GetCheckedMedia()
             If m.ListExists Then
                 Settings.Feeds.Favorite.Add(m)
-                If sender Is BTT_FEED_ADD_FAV_REMOVE Then FeedRemoveCheckedMedia(m,,, {FeedSpecial.FavoriteName})
+                If sender Is BTT_FEED_ADD_FAV_REMOVE Then FeedRemoveCheckedMedia(m,,, {FeedSpecial.FavoriteName}, True)
             End If
         End Sub
         Private Sub BTT_FEED_REMOVE_FAV_Click(sender As Object, e As EventArgs) Handles BTT_FEED_REMOVE_FAV.Click
             Dim m As IEnumerable(Of UserMediaD) = GetCheckedMedia()
             If m.ListExists Then
                 Settings.Feeds.Favorite.Remove(m)
-                If FeedMode = FeedModes.Special Then FeedRemoveCheckedMedia(m, {FeedSpecial.FavoriteName}.ToList)
+                If FeedMode = FeedModes.Special Then FeedRemoveCheckedMedia(m, {FeedSpecial.FavoriteName}.ToList,,, False)
             End If
         End Sub
         Private Sub BTT_FEED_ADD_SPEC_Click(sender As Object, e As EventArgs) Handles BTT_FEED_ADD_SPEC.Click, BTT_FEED_ADD_SPEC_REMOVE.Click
@@ -1020,7 +1028,7 @@ Namespace DownloadObjects
                                                      f.Add(c)
                                                  End Sub)
                 End With
-                If sender Is BTT_FEED_ADD_SPEC_REMOVE Then FeedRemoveCheckedMedia(c,,, names)
+                If sender Is BTT_FEED_ADD_SPEC_REMOVE Then FeedRemoveCheckedMedia(c,,, names, True)
                 names.Clear()
             Else
                 MsgBoxE({"You haven't selected media to add to your feed(s)", "Add to feed(s)"}, vbExclamation)
@@ -1036,7 +1044,7 @@ Namespace DownloadObjects
                                                      f.Remove(c)
                                                  End Sub)
                 End With
-                If FeedMode = FeedModes.Special Then FeedRemoveCheckedMedia(c, names)
+                If FeedMode = FeedModes.Special Then FeedRemoveCheckedMedia(c, names,,, False)
             Else
                 MsgBoxE({"You haven't selected media to remove from your feed(s)", "Remove from feed(s)"}, vbExclamation)
             End If
@@ -1408,8 +1416,10 @@ Namespace DownloadObjects
                 ErrorsDescriber.Execute(EDP.LogMessageValue, ex, "Download subscription media")
             End Try
         End Sub
-        Private Sub FeedMedia_FeedAddWithRemove(ByVal Sender As FeedMedia, ByVal Feeds As IEnumerable(Of String), ByVal Media As UserMediaD, ByVal RemoveOperation As Boolean)
-            FeedRemoveCheckedMedia({Media},, False, Feeds, RemoveOperation)
+        Private Sub FeedMedia_FeedRemoveCheckedMedia(ByVal Sender As FeedMedia, ByVal Media As UserMediaD, ByVal Names As IEnumerable(Of String),
+                                                     ByVal IsOverriddenNames As Boolean, ByVal IsAddAndRemove As Boolean)
+            FeedRemoveCheckedMedia({Media}, CObj(If(IsOverriddenNames, Names.ListIfNothing, Nothing)), False,
+                                   CObj(If(IsOverriddenNames, Nothing, Names)), IsAddAndRemove)
         End Sub
 #End Region
 #Region "Delete / Remove"
@@ -1616,7 +1626,7 @@ Namespace DownloadObjects
                                                   AddHandler .MediaDownload, AddressOf FeedMedia_Download
                                                   AddHandler .MediaMove, AddressOf FeedMedia_MediaMove
                                                   AddHandler .MediaCopy, AddressOf FeedMedia_MediaCopy
-                                                  AddHandler .FeedAddWithRemove, AddressOf FeedMedia_FeedAddWithRemove
+                                                  AddHandler .FeedRemoveCheckedMedia, AddressOf FeedMedia_FeedRemoveCheckedMedia
                                               End With
                                               If de.Data.Type = UTypes.Text OrElse de.Data.PostTextFile.IsEmptyString Then Exit For
                                           Next

SCrawler/Download/Feed/FeedMedia.vb

@@ -12,6 +12,7 @@ Imports PersonalUtilities.Forms
 Imports PersonalUtilities.Tools
 Imports SCrawler.API.Base
 Imports UserMediaD = SCrawler.DownloadObjects.TDownloader.UserMediaD
+Imports ImageRenderer2 = SCrawler.UserImage.ImageRenderer2
 Namespace DownloadObjects
     <ToolboxItem(False), DesignTimeVisible(False)>
     Public Class FeedMedia
@@ -20,7 +21,8 @@ Namespace DownloadObjects
         Friend Event MediaDeleted(ByVal Sender As Object)
         Friend Event MediaDeletedText(ByVal Sender As Object)
         Friend Event MediaDownload As EventHandler
-        Friend Event FeedAddWithRemove(ByVal Sender As FeedMedia, ByVal Feeds As IEnumerable(Of String), ByVal Media As UserMediaD, ByVal RemoveOperation As Boolean)
+        Friend Event FeedRemoveCheckedMedia(ByVal Sender As FeedMedia, ByVal Media As UserMediaD, ByVal Names As IEnumerable(Of String),
+                                            ByVal IsOverriddenNames As Boolean, ByVal IsAddAndRemove As Boolean)
         Friend Event MediaMove As MediaMoveCopyEventHandler
         Friend Event MediaCopy As MediaMoveCopyEventHandler
 #End Region
@@ -165,42 +167,6 @@ Namespace DownloadObjects
         Public Sub New()
             InitializeComponent()
         End Sub
-        Private Class ImageRenderer2 : Inherits ImageRenderer
-            Friend NativeFormat As String = Nothing
-            Friend ImgErr As Exception = Nothing
-            Friend Sub New(ByVal ImgPath As SFile, Optional ByVal e As ErrorsDescriber = Nothing)
-                MyBase.New()
-                Try
-                    If ImgPath.Exists(SFO.File, False) Then
-                        OriginalImageBytes = SFile.GetBytes(ImgPath, EDP.ThrowException)
-                        Try
-                            OriginalImage = GetImage(OriginalImageBytes)
-                        Catch exInternal As Exception
-                            HasError = True
-                            ImgErr = exInternal
-                            NativeFormat = GetTrueFormat(OriginalImageBytes, EDP.ReturnValue)
-                        End Try
-                    End If
-                    Address = ImgPath
-                Catch ex As Exception
-                    HasError = True
-                    NativeFormat = GetTrueFormat(OriginalImageBytes, EDP.ReturnValue)
-                    If Not e.Exists Then e = EDP.ThrowException
-                    ErrorsDescriber.Execute(e, ex, $"ImageRenderer2.New({ImgPath})")
-                End Try
-            End Sub
-            Friend Shared Function GetTrueFormat(ByVal Img() As Byte, Optional ByVal e As ErrorsDescriber = Nothing) As String
-                Try
-                    Using ms As New MemoryStream(Img, 0, Img.Length)
-                        Return System.Windows.Media.Imaging.BitmapDecoder.Create(ms, Windows.Media.Imaging.BitmapCreateOptions.PreservePixelFormat,
-                                                                                 Windows.Media.Imaging.BitmapCacheOption.OnLoad).Metadata.Format
-                    End Using
-                Catch ex As Exception
-                    If Not e.Exists Then e = EDP.ThrowException
-                    Return ErrorsDescriber.Execute(e, ex, "[ImageRenderer2.GetTrueFormat()]")
-                End Try
-            End Function
-        End Class
         Friend Sub New(ByVal Media As UserMediaD, ByVal Width As Integer, ByVal Height As Integer, ByVal IsSession As Boolean, ByVal ExtractText As Boolean)
             Try
                 InitializeComponent()
@@ -435,11 +401,11 @@ Namespace DownloadObjects
         End Function
         Private Sub Feed_SPEC_ADD_REMOVE(ByVal Source As ToolStripMenuItem, ByVal e As EventArgs)
             Dim f As FeedSpecial = Feed_SPEC_ADD_Impl(Source)
-            If Not f Is Nothing Then RaiseEvent FeedAddWithRemove(Me, {f.Name}, Media, False)
+            If Not f Is Nothing Then RaiseEvent FeedRemoveCheckedMedia(Me, Media, {f.Name}, False, True)
         End Sub
         Private Sub Feed_SPEC_REMOVE(ByVal Source As ToolStripMenuItem, ByVal e As EventArgs)
             Dim f As FeedSpecial = Source.Tag
-            If Not f Is Nothing AndAlso Not f.Disposed Then f.Remove(Media) : RaiseEvent FeedAddWithRemove(Me, {f.Name}, Media, True)
+            If Not f Is Nothing AndAlso Not f.Disposed Then f.Remove(Media) : RaiseEvent FeedRemoveCheckedMedia(Me, Media, {f.Name}, True, False)
         End Sub
 #End Region
 #Region "Dispose"
@@ -598,14 +564,14 @@ Namespace DownloadObjects
             With Settings.Feeds.Favorite
                 If Not .Contains(Media) Then .Add(Media)
                 BTT_FEED_ADD_FAV.ControlChangeColor(True, False)
-                If sender Is BTT_FEED_ADD_FAV_REMOVE Then RaiseEvent FeedAddWithRemove(Me, {FeedSpecial.FavoriteName}, Media, False)
+                If sender Is BTT_FEED_ADD_FAV_REMOVE Then RaiseEvent FeedRemoveCheckedMedia(Me, Media, {FeedSpecial.FavoriteName}, False, True)
             End With
         End Sub
         Private Sub BTT_FEED_ADD_SPEC_Click(sender As Object, e As EventArgs) Handles BTT_FEED_ADD_SPEC.Click, BTT_FEED_ADD_SPEC_REMOVE.Click
             With FeedSpecialCollection.ChooseFeeds(True)
                 If .ListExists Then
                     .ForEach(Sub(f) f.Add(Media))
-                    If sender Is BTT_FEED_ADD_SPEC_REMOVE Then RaiseEvent FeedAddWithRemove(Me, .Select(Function(f) f.Name), Media, False)
+                    If sender Is BTT_FEED_ADD_SPEC_REMOVE Then RaiseEvent FeedRemoveCheckedMedia(Me, Media, .Select(Function(f) f.Name), False, True)
                 End If
             End With
         End Sub
@@ -613,14 +579,14 @@ Namespace DownloadObjects
             With Settings.Feeds.Favorite
                 If .Contains(Media) Then .Remove(Media)
                 BTT_FEED_ADD_FAV.ControlChangeColor(True)
-                RaiseEvent FeedAddWithRemove(Me, {FeedSpecial.FavoriteName}, Media, True)
+                RaiseEvent FeedRemoveCheckedMedia(Me, Media, {FeedSpecial.FavoriteName}, True, False)
             End With
         End Sub
         Private Sub BTT_FEED_REMOVE_SPEC_Click(sender As Object, e As EventArgs) Handles BTT_FEED_REMOVE_SPEC.Click
             With FeedSpecialCollection.ChooseFeeds(False)
                 If .ListExists Then
                     .ForEach(Sub(f) f.Remove(Media))
-                    RaiseEvent FeedAddWithRemove(Me, .Select(Function(f) f.Name), Media, True)
+                    RaiseEvent FeedRemoveCheckedMedia(Me, Media, .Select(Function(f) f.Name), True, False)
                 End If
             End With
         End Sub

SCrawler/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2026.1.24.0")>
-<Assembly: AssemblyFileVersion("2026.1.24.0")>
+<Assembly: AssemblyVersion("2026.3.23.0")>
+<Assembly: AssemblyFileVersion("2026.3.23.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler/SCrawler.vbproj

@@ -194,6 +194,7 @@
     <Compile Include="API\Base\TokenBatch.vb" />
     <Compile Include="API\Base\YTDLP.vb" />
     <Compile Include="API\Bluesky\Declarations.vb" />
+    <Compile Include="API\Bluesky\EditorExchangeOptions.vb" />
     <Compile Include="API\Bluesky\M3U8.vb" />
     <Compile Include="API\Bluesky\SiteSettings.vb" />
     <Compile Include="API\Bluesky\UserData.vb" />

SCrawler/UserImage.vb

@@ -150,4 +150,45 @@ Friend Class UserImage : Inherits ImageRenderer
     Private Shared Function ConvertWebpTryImageMagick(ByVal InitFile As SFile, ByVal DestFile As SFile) As Boolean
         Return ImageRendererExt.ConvertWebp(InitFile, DestFile, EDP.SendToLog + EDP.ReturnValue)
     End Function
+    Friend Class ImageRenderer2 : Inherits ImageRenderer
+        Friend NativeFormat As String = Nothing
+        Friend ImgErr As Exception = Nothing
+        Friend ReadOnly Property IsWebP As Boolean
+            Get
+                Return NativeFormat.IfNullOrEmpty(ExtJpg).ToLower = ExtWebp
+            End Get
+        End Property
+        Friend Sub New(ByVal ImgPath As SFile, Optional ByVal e As ErrorsDescriber = Nothing)
+            MyBase.New()
+            Try
+                If ImgPath.Exists(SFO.File, False) Then
+                    OriginalImageBytes = SFile.GetBytes(ImgPath, EDP.ThrowException)
+                    Try
+                        OriginalImage = GetImage(OriginalImageBytes)
+                    Catch exInternal As Exception
+                        HasError = True
+                        ImgErr = exInternal
+                        NativeFormat = GetTrueFormat(OriginalImageBytes, EDP.ReturnValue)
+                    End Try
+                End If
+                Address = ImgPath
+            Catch ex As Exception
+                HasError = True
+                NativeFormat = GetTrueFormat(OriginalImageBytes, EDP.ReturnValue)
+                If Not e.Exists Then e = EDP.ThrowException
+                ErrorsDescriber.Execute(e, ex, $"ImageRenderer2.New({ImgPath})")
+            End Try
+        End Sub
+        Friend Shared Function GetTrueFormat(ByVal Img() As Byte, Optional ByVal e As ErrorsDescriber = Nothing) As String
+            Try
+                Using ms As New MemoryStream(Img, 0, Img.Length)
+                    Return System.Windows.Media.Imaging.BitmapDecoder.Create(ms, Windows.Media.Imaging.BitmapCreateOptions.PreservePixelFormat,
+                                                                             Windows.Media.Imaging.BitmapCacheOption.OnLoad).Metadata.Format
+                End Using
+            Catch ex As Exception
+                If Not e.Exists Then e = EDP.ThrowException
+                Return ErrorsDescriber.Execute(e, ex, "[ImageRenderer2.GetTrueFormat()]")
+            End Try
+        End Function
+    End Class
 End Class