2023.11.13.1

PornHub: add playlists downloading

SCrawler/API/Base/Structures.vb

@@ -18,6 +18,7 @@ Namespace API.Base
             Tags = 2
             Categories = 3
             Pornstars = 4
+            Playlists = 5
         End Enum
         Friend Structure UserMedia : Implements IUserMedia, IEquatable(Of UserMedia), IEContainerProvider
 #Region "XML Names"

SCrawler/API/Base/UserDataBase.vb

@@ -169,6 +169,11 @@ Namespace API.Base
         Protected Const Name_UseMD5Comparison As String = "UseMD5Comparison"
         Protected Const Name_RemoveExistingDuplicates As String = "RemoveExistingDuplicates"
         Protected Const Name_StartMD5Checked As String = "StartMD5Checked"
+#Region "Additional names"
+        Protected Const Name_SiteMode As String = "SiteMode"
+        Protected Const Name_TrueName As String = "TrueName"
+        Protected Const Name_Arguments As String = "Arguments"
+#End Region
 #End Region
 #Region "Declarations"
 #Region "Host, Site, Progress"

SCrawler/API/Mastodon/UserData.vb

@@ -18,7 +18,6 @@ Namespace API.Mastodon
     Friend Class UserData : Inherits Twitter.UserData
 #Region "XML names"
         Private Const Name_UserDomain As String = "UserDomain"
-        Private Const Name_TrueName As String = "TrueName"
 #End Region
 #Region "Declarations"
         Private _UserDomain As String = String.Empty

SCrawler/API/PornHub/Declarations.vb

@@ -24,6 +24,7 @@ Namespace API.PornHub
                                                                 0, RegexOptions.Singleline, RegexReturn.List, EDP.ReturnValue, UnicodeHexConverter)
         Friend ReadOnly RegexVideo_Video_VideoKey As RParams = RParams.DMS("viewkey=([\w\d]+)", 1, EDP.ReturnValue)
         Friend ReadOnly RegexVideoPageTitle As RParams = RParams.DMS("meta (property|name)=""[^:]+?:title"" content=""([^""]+)""", 2, EDP.ReturnValue)
+        Friend ReadOnly RegexDataToken As RParams = RParams.DMS("data-token=""([^""]+)", 1, EDP.ReturnValue)
 #End Region
 #Region "Declarations M3U8"
         Friend ReadOnly Regex_M3U8_FilesList As RParams = RParams.DM("RESOLUTION=\d+x(\d+).*?[\r\n]*?(.+?m3u8.*)", 0, RegexReturn.List, EDP.ReturnValue)

SCrawler/API/PornHub/UserData.vb

@@ -29,7 +29,7 @@ Namespace API.PornHub
         Private Const Name_DownloadFavorite As String = "DownloadFavorite"
         Private Const Name_DownloadGifs As String = "DownloadGifs"
         Private Const Name_DownloadPhotoOnlyFromModelHub As String = "DownloadPhotoOnlyFromModelHub"
-        Private Const Name_IsUser As String = "IsUser"
+        <Obsolete> Private Const Name_IsUser As String = "IsUser"
 #End Region
 #Region "Structures"
         Private Structure FlashVar : Implements IRegExCreator
@@ -117,6 +117,8 @@ Namespace API.PornHub
         Private Const PersonTypeUser As String = "users"
         Private Const PersonTypePornstar As String = "pornstar"
         Private Const PersonTypeCannel As String = "channels"
+        Private Const PersonTypePlaylist As String = "playlist"
+        Private Const PlaylistsLabelName As String = "Playlist"
 #End Region
 #Region "Person"
         Friend Property PersonType As String
@@ -133,7 +135,7 @@ Namespace API.PornHub
 #Region "Advanced fields"
         Friend Overrides ReadOnly Property FeedIsUser As Boolean
             Get
-                Return IsUser
+                Return IsUser Or SiteMode = SiteModes.Playlists
             End Get
         End Property
         Private Property PhotoPageModel As PhotoPageModels = PhotoPageModels.Undefined
@@ -144,12 +146,12 @@ Namespace API.PornHub
         Friend Property DownloadFavorite As Boolean = False
         Friend Property DownloadGifs As Boolean
         Friend Property DownloadPhotoOnlyFromModelHub As Boolean = True
-        Private _IsUser As Boolean = True
         Friend Overrides ReadOnly Property IsUser As Boolean
             Get
-                Return _IsUser
+                Return SiteMode = SiteModes.User
             End Get
         End Property
+        Friend Property SiteMode As SiteModes = SiteModes.User
         Friend Property QueryString As String
             Get
                 If IsUser Then
@@ -164,7 +166,7 @@ Namespace API.PornHub
         End Property
         Friend Overrides ReadOnly Property SpecialLabels As IEnumerable(Of String)
             Get
-                Return {SearchRequestLabelName}
+                Return {SearchRequestLabelName, PlaylistsLabelName}
             End Get
         End Property
 #End Region
@@ -204,26 +206,32 @@ Namespace API.PornHub
 #Region "Loader"
         Private Function UpdateUserOptions(Optional ByVal Force As Boolean = False, Optional ByVal NewUrl As String = Nothing) As Boolean
 
-            If Not Force OrElse (Not IsUser AndAlso Not NewUrl.IsEmptyString AndAlso MyFileSettings.Exists) Then
+            If Not Force OrElse (Not IsUser AndAlso Not SiteMode = SiteModes.Playlists AndAlso Not NewUrl.IsEmptyString AndAlso MyFileSettings.Exists) Then
                 Dim eObj As Plugin.ExchangeOptions = Nothing
                 If Force Then eObj = MySettings.IsMyUser(NewUrl)
                 If (Force And Not eObj.UserName.IsEmptyString) Or (Not Force And Not Name.IsEmptyString And NameTrue.IsEmptyString) Then
                     If Not If(Force, eObj.Options, Options).IsEmptyString Then
-                        If IsUser And Force Then
+                        If (IsUser Or SiteMode = SiteModes.Playlists) And Force Then
                             Return False
                         Else
-                            _IsUser = False
+                            SiteMode = SiteModes.Search
                             Options = If(Force, eObj.Options, Options)
+                            If Options.ToLower.StartsWith(PersonTypePlaylist) Then
+                                SiteMode = SiteModes.Playlists
+                                NameTrue = Options.ToLower.Replace(PersonTypePlaylist, String.Empty).StringTrim.TrimStart("/")
+                            Else
                                 NameTrue = Options
+                            End If
                             If Not Force Then
-                                Settings.Labels.Add(SearchRequestLabelName)
+                                Dim l$ = IIf(SiteMode = SiteModes.Playlists, PlaylistsLabelName, SearchRequestLabelName)
-                                Labels.ListAddValue(SearchRequestLabelName, LNC)
+                                Settings.Labels.Add(l)
+                                Labels.ListAddValue(l, LNC)
                                 Labels.Sort()
                                 Return True
                             End If
                         End If
                     Else
-                        _IsUser = True
+                        SiteMode = SiteModes.User
                         Dim n$() = Name.Split("_")
                         If n.ListExists(2) Then
                             NameTrue = Name.Replace($"{n(0)}_", String.Empty)
@@ -247,7 +255,14 @@ Namespace API.PornHub
                     DownloadFavorite = .Value(Name_DownloadFavorite).FromXML(Of Boolean)(False)
                     DownloadGifs = .Value(Name_DownloadGifs).FromXML(Of Integer)(False)
                     DownloadPhotoOnlyFromModelHub = .Value(Name_DownloadPhotoOnlyFromModelHub).FromXML(Of Boolean)(True)
-                    _IsUser = .Value(Name_IsUser).FromXML(Of Boolean)(True)
+                    If .Contains(Name_SiteMode) Then
+                        SiteMode = .Value(Name_SiteMode).FromXML(Of Integer)(SiteModes.User)
+                    Else
+                        'TODELETE: PornHub 'IsUser' 20231113
+#Disable Warning BC40008
+                        SiteMode = IIf(.Value(Name_IsUser).FromXML(Of Boolean)(True), SiteModes.User, SiteModes.Search)
+#Enable Warning
+                    End If
                     UpdateUserOptions()
                 Else
                     If UpdateUserOptions() Then .Value(Name_LabelsName) = LabelsString
@@ -261,7 +276,7 @@ Namespace API.PornHub
                     .Add(Name_DownloadFavorite, DownloadFavorite.BoolToInteger)
                     .Add(Name_DownloadGifs, DownloadGifs.BoolToInteger)
                     .Add(Name_DownloadPhotoOnlyFromModelHub, DownloadPhotoOnlyFromModelHub.BoolToInteger)
-                    .Add(Name_IsUser, IsUser.BoolToInteger)
+                    .Add(Name_SiteMode, CInt(SiteMode))
 
                     'Debug.WriteLine(GetNonUserUrl(0))
                     'Debug.WriteLine(GetNonUserUrl(2))
@@ -271,8 +286,11 @@ Namespace API.PornHub
 #End Region
 #Region "Downloading"
 #Region "Download override"
+        Private Const PlayListUrlPattern As String = "https://www.pornhub.com/playlist/viewChunked?id={0}&token={1}&page={2}"
+        Private PlaylistToken As String = String.Empty
         Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
             Try
+                PlaylistToken = String.Empty
                 Responser.ResetStatus()
 
                 If IsSavedPosts Then
@@ -283,7 +301,11 @@ Namespace API.PornHub
                 Dim limit% = If(DownloadTopCount, -1)
                 If DownloadVideos Then
 
-                    If IsSavedPosts Or Not IsUser Or PersonType = PersonTypeUser Then
+                    If SiteMode = SiteModes.Playlists Then
+                        Responser.Mode = Responser.Modes.Default
+                        GetPlaylistToken(Token)
+                        DownloadUserVideos(1, VideoTypes.Favorite, False, Token)
+                    ElseIf IsSavedPosts Or Not IsUser Or PersonType = PersonTypeUser Then
                         DownloadUserVideos(1, VideoTypes.Favorite, False, Token)
                     Else
                         If DownloadUploaded Then
@@ -365,6 +387,9 @@ Namespace API.PornHub
                         Throw New ArgumentException($"Type '{Type}' is not implemented in the video download function", "Type")
                     End If
                     If Page > 1 Then URL &= $"?page={Page}"
+                ElseIf SiteMode = SiteModes.Playlists Then
+                    If PlaylistToken.IsEmptyString Then Throw New ArgumentNullException("PlaylistToken", "Unable to get 'PlaylistToken'")
+                    URL = String.Format(PlayListUrlPattern, NameTrue, PlaylistToken, Page)
                 Else
                     URL = GetNonUserUrl(Page)
                 End If
@@ -375,7 +400,7 @@ Namespace API.PornHub
                 Dim r$ = Responser.GetResponse(URL)
                 If Not r.IsEmptyString Then
                     Dim l As List(Of UserVideo) = RegexFields(Of UserVideo)(r, {RegexUserVideos}, {6, 7, 3, 10})
-                    If l.ListExists Then l = l.ListTake(3, l.Count).ToList
+                    If l.ListExists And Not SiteMode = SiteModes.Playlists Then l = l.ListTake(3, l.Count).ToList
                     If l.ListExists Then
                         If IsUser Then
                             If Type = VideoTypes.Favorite Then
@@ -419,6 +444,15 @@ Namespace API.PornHub
                 ProgressPre.Perform()
             End Try
         End Sub
+        Private Sub GetPlaylistToken(ByVal Token As CancellationToken)
+            Dim URL$ = GetNonUserUrl(0)
+            Try
+                Dim r$ = Responser.GetResponse(URL)
+                If Not r.IsEmptyString Then PlaylistToken = RegexReplace(r, RegexDataToken)
+            Catch ex As Exception
+                ProcessException(ex, Token, $"token getting error [{URL}]")
+            End Try
+        End Sub
 #End Region
 #Region "Download GIF"
         Private Sub DownloadUserGifs(ByVal Token As CancellationToken)

SCrawler/API/Reddit/UserData.vb

@@ -23,9 +23,6 @@ Imports CView = SCrawler.API.Reddit.IRedditView.View
 Imports CPeriod = SCrawler.API.Reddit.IRedditView.Period
 Namespace API.Reddit
     Friend Class UserData : Inherits UserDataBase : Implements IChannelLimits, IRedditView
-#Region "XML names"
-        Private Const Name_TrueName As String = "TrueName"
-#End Region
 #Region "Declarations"
         Private Const CannelsLabelName As String = "Channels"
         Friend Const CannelsLabelName_ChannelsForm As String = "RChannels"

SCrawler/API/ThisVid/UserData.vb

@@ -20,9 +20,6 @@ Namespace API.ThisVid
         Private Const Name_DownloadPrivate As String = "DownloadPrivate"
         Private Const Name_DownloadFavourite As String = "DownloadFavourite"
         Private Const Name_DifferentFolders As String = "DifferentFolders"
-        Private Const Name_TrueName As String = "TrueName"
-        Private Const Name_SiteMode As String = "SiteMode"
-        Private Const Name_Arguments As String = "Arguments"
 #End Region
 #Region "Structures"
         Private Structure Album : Implements IRegExCreator

SCrawler/API/XVIDEOS/UserData.vb

@@ -17,9 +17,6 @@ Imports UTypes = SCrawler.API.Base.UserMedia.Types
 Namespace API.XVIDEOS
     Friend Class UserData : Inherits UserDataBase
 #Region "XML names"
-        Private Const Name_SiteMode As String = "SiteMode"
-        Private Const Name_TrueName As String = "TrueName"
-        Private Const Name_Arguments As String = "Arguments"
         Private Const Name_PersonType As String = "PersonType"
 #End Region
 #Region "Structures"

SCrawler/API/Xhamster/UserData.vb

@@ -18,10 +18,7 @@ Imports UTypes = SCrawler.API.Base.UserMedia.Types
 Namespace API.Xhamster
     Friend Class UserData : Inherits UserDataBase
 #Region "XML names"
-        Private Const Name_TrueName As String = "TrueName"
         Private Const Name_Gender As String = "Gender"
-        Private Const Name_SiteMode As String = "SiteMode"
-        Private Const Name_Arguments As String = "Arguments"
 #End Region
 #Region "Declarations"
         Friend Overrides ReadOnly Property FeedIsUser As Boolean