2022.11.16.0

Add sites: PornHub, XHamster
Add saved xvideos posts downloading
PluginProvider: added TaskGroup attribute; added IUserMedia inteface; changed PluginUserMedia to IUserMedia in interface declarations; changed 'User' String to IPluginContentProvider in ISiteSettings sinterface
Added update the 'LOG' button at the end of the ProfileSaved download function
API.Base: added 'IUserMedia' compatibility for 'UserMedia'; moved 'GetImage' from 'UserPost' to 'ChannelsViewForm'; update constants in UserDataBase; updated UserDataBase to new UserInfo environment.
API.Instagram.UserData: fixed date issue
API.Reddit.SiteSettings: update user patterns
API.Twitter.Declarations: moved provider here from MainFrame
UserDataBind: updated to new UserInfo environment
ActiveDownloadingProgress: updated form rendering
AutoDownloader: added SpecialDelay
TDownloader: added 'Suspended' option; updated for TaskGroups
CollectionEditorForm: fixed order bug
LabelsForm: remove old stuff
UserEditorForm: added collection editing
MainFrame: improve label selection
Add import users
Added the ability to create a virtual collection and add a virtual user to a real collection
SettingsCLS: improve users loading

SCrawler/API/Twitter/Declarations.vb

@@ -6,14 +6,21 @@
 '
 ' This program is distributed in the hope that it will be useful,
 ' but WITHOUT ANY WARRANTY
+Imports System.Globalization
 Imports PersonalUtilities.Functions.XML.Base
 Imports PersonalUtilities.Functions.RegularExpressions
 Namespace API.Twitter
     Friend Module Declarations
         Friend Const TwitterSite As String = "Twitter"
-        Friend DateProvider As New ADateTime(ADateTime.Formats.BaseDateTime)
+        Friend ReadOnly DateProvider As ADateTime = GetDateProvider()
         Friend ReadOnly VideoNode As NodeParams() = {New NodeParams("video_info", True, True, True, True, 10)}
         Friend ReadOnly VideoSizeRegEx As RParams = RParams.DMS("\d+x(\d+)", 1, EDP.ReturnValue)
         Friend ReadOnly UserIdRegEx As RParams = RParams.DMS("user_id.:.(\d+)", 1, EDP.ReturnValue)
+        Private Function GetDateProvider() As ADateTime
+            Dim n As DateTimeFormatInfo = CultureInfo.GetCultureInfo("en-us").DateTimeFormat.Clone
+            n.FullDateTimePattern = "ddd MMM dd HH:mm:ss +ffff yyyy"
+            n.TimeSeparator = String.Empty
+            Return New ADateTime(DirectCast(n.Clone, DateTimeFormatInfo)) With {.DateTimeStyle = DateTimeStyles.AssumeUniversal}
+        End Function
     End Module
 End Namespace

SCrawler/API/Twitter/SiteSettings.vb

@@ -9,8 +9,9 @@
 Imports SCrawler.API.Base
 Imports SCrawler.Plugin
 Imports SCrawler.Plugin.Attributes
-Imports PersonalUtilities.Tools.WEB
 Imports PersonalUtilities.Functions.RegularExpressions
+Imports PersonalUtilities.Tools.Web.Clients
+Imports PersonalUtilities.Tools.Web.Cookies
 Namespace API.Twitter
     <Manifest("AndyProgram_Twitter"), SavedPosts>
     Friend Class SiteSettings : Inherits SiteSettingsBase
@@ -31,7 +32,7 @@ Namespace API.Twitter
         Private ReadOnly Property Auth As PropertyValue
         <PropertyOption(AllowNull:=False, ControlText:="Token", ControlToolTip:="Set token from [x-csrf-token] response header")>
         Private ReadOnly Property Token As PropertyValue
-        <PropertyOption(ControlText:="Saved posts user name", ControlToolTip:="Personal profile username", LeftOffset:=120), PXML>
+        <PropertyOption(ControlText:="Saved posts user", ControlToolTip:="Personal profile username"), PXML>
         Friend ReadOnly Property SavedPostsUserName As PropertyValue
         Friend Overrides ReadOnly Property Responser As Response
         Friend Sub New()
@@ -45,10 +46,8 @@ Namespace API.Twitter
                 If .File.Exists Then
                     If EncryptCookies.CookiesEncrypted Then .CookiesEncryptKey = SettingsCLS.CookieEncryptKey
                     .LoadSettings()
-                    With .Headers
-                        If .ContainsKey(Header_Authorization) Then a = .Item(Header_Authorization)
-                        If .ContainsKey(Header_Token) Then t = .Item(Header_Token)
-                    End With
+                    a = .HeadersValue(Header_Authorization)
+                    t = .HeadersValue(Header_Token)
                 Else
                     .ContentType = "application/json"
                     .Accept = "*/*"
@@ -56,17 +55,15 @@ Namespace API.Twitter
                     .Cookies = New CookieKeeper(.CookiesDomain) With {.EncryptKey = SettingsCLS.CookieEncryptKey}
                     .CookiesEncryptKey = SettingsCLS.CookieEncryptKey
                     .Decoders.Add(SymbolsConverter.Converters.Unicode)
-                    With .Headers
-                        .Add("sec-ch-ua", " Not;A Brand"";v=""99"", ""Google Chrome"";v=""91"", ""Chromium"";v=""91""")
-                        .Add("sec-ch-ua-mobile", "?0")
-                        .Add("sec-fetch-dest", "empty")
-                        .Add("sec-fetch-mode", "cors")
-                        .Add("sec-fetch-site", "same-origin")
-                        .Add(Header_Token, String.Empty)
-                        .Add("x-twitter-active-user", "yes")
-                        .Add("x-twitter-auth-type", "OAuth2Session")
-                        .Add(Header_Authorization, String.Empty)
-                    End With
+                    .HeadersAdd("sec-ch-ua", " Not;A Brand"";v=""99"", ""Google Chrome"";v=""91"", ""Chromium"";v=""91""")
+                    .HeadersAdd("sec-ch-ua-mobile", "?0")
+                    .HeadersAdd("sec-fetch-dest", "empty")
+                    .HeadersAdd("sec-fetch-mode", "cors")
+                    .HeadersAdd("sec-fetch-site", "same-origin")
+                    .HeadersAdd(Header_Token, String.Empty)
+                    .HeadersAdd("x-twitter-active-user", "yes")
+                    .HeadersAdd("x-twitter-auth-type", "OAuth2Session")
+                    .HeadersAdd(Header_Authorization, String.Empty)
                     .SaveSettings()
                 End If
             End With
@@ -87,8 +84,8 @@ Namespace API.Twitter
                     Case NameOf(Token) : f = Header_Token
                 End Select
                 If Not f.IsEmptyString Then
-                    If Responser.Headers.Count > 0 AndAlso Responser.Headers.ContainsKey(f) Then Responser.Headers.Remove(f)
-                    If Not CStr(Value).IsEmptyString Then Responser.Headers.Add(f, CStr(Value))
+                    Responser.HeadersRemove(f)
+                    If Not CStr(Value).IsEmptyString Then Responser.HeadersAdd(f, CStr(Value))
                     Responser.SaveSettings()
                 End If
             End If
@@ -103,8 +100,8 @@ Namespace API.Twitter
         Friend Overrides Function GetSpecialData(ByVal URL As String, ByVal Path As String, ByVal AskForPath As Boolean) As IEnumerable
             Return UserData.GetVideoInfo(URL, Responser)
         End Function
-        Friend Overrides Function GetUserPostUrl(ByVal UserID As String, ByVal PostID As String) As String
-            Return $"https://twitter.com/{UserID}/status/{PostID}"
+        Friend Overrides Function GetUserPostUrl(ByVal User As UserDataBase, ByVal Media As UserMedia) As String
+            Return $"https://twitter.com/{User.Name}/status/{Media.Post.ID}"
         End Function
         Friend Overrides Function BaseAuthExists() As Boolean
             Return If(Responser.Cookies?.Count, 0) > 0 And ACheck(Token.Value) And ACheck(Auth.Value)

SCrawler/API/Twitter/UserData.vb

@@ -11,8 +11,8 @@ Imports System.Threading
 Imports SCrawler.API.Base
 Imports PersonalUtilities.Functions.XML
 Imports PersonalUtilities.Functions.RegularExpressions
-Imports PersonalUtilities.Tools.WEB
-Imports PersonalUtilities.Tools.WebDocuments.JSON
+Imports PersonalUtilities.Tools.Web.Clients
+Imports PersonalUtilities.Tools.Web.Documents.JSON
 Imports UStates = SCrawler.API.Base.UserMedia.States
 Namespace API.Twitter
     Friend Class UserData : Inherits UserDataBase