2023.11.13.1

PornHub: add playlists downloading
2026-03-14 15:52:18 +00:00 · 2023-11-13 23:13:53 +03:00
parent f08a5f9259
commit 96705f1c59
9 changed files with 57 additions and 29 deletions
--- a/SCrawler/API/Base/Structures.vb
+++ b/SCrawler/API/Base/Structures.vb
@@ -18,6 +18,7 @@ Namespace API.Base
            Tags = 2
            Categories = 3
            Pornstars = 4
+            Playlists = 5
        End Enum
        Friend Structure UserMedia : Implements IUserMedia, IEquatable(Of UserMedia), IEContainerProvider
 #Region "XML Names"
--- a/SCrawler/API/Base/UserDataBase.vb
+++ b/SCrawler/API/Base/UserDataBase.vb
@@ -169,6 +169,11 @@ Namespace API.Base
        Protected Const Name_UseMD5Comparison As String = "UseMD5Comparison"
        Protected Const Name_RemoveExistingDuplicates As String = "RemoveExistingDuplicates"
        Protected Const Name_StartMD5Checked As String = "StartMD5Checked"
+#Region "Additional names"
+        Protected Const Name_SiteMode As String = "SiteMode"
+        Protected Const Name_TrueName As String = "TrueName"
+        Protected Const Name_Arguments As String = "Arguments"
+#End Region
 #End Region
 #Region "Declarations"
 #Region "Host, Site, Progress"
--- a/SCrawler/API/Mastodon/UserData.vb
+++ b/SCrawler/API/Mastodon/UserData.vb
@@ -18,7 +18,6 @@ Namespace API.Mastodon
    Friend Class UserData : Inherits Twitter.UserData
 #Region "XML names"
        Private Const Name_UserDomain As String = "UserDomain"
-        Private Const Name_TrueName As String = "TrueName"
 #End Region
 #Region "Declarations"
        Private _UserDomain As String = String.Empty
--- a/SCrawler/API/PornHub/Declarations.vb
+++ b/SCrawler/API/PornHub/Declarations.vb
@@ -24,6 +24,7 @@ Namespace API.PornHub
                                                                0, RegexOptions.Singleline, RegexReturn.List, EDP.ReturnValue, UnicodeHexConverter)
        Friend ReadOnly RegexVideo_Video_VideoKey As RParams = RParams.DMS("viewkey=([\w\d]+)", 1, EDP.ReturnValue)
        Friend ReadOnly RegexVideoPageTitle As RParams = RParams.DMS("meta (property|name)=""[^:]+?:title"" content=""([^""]+)""", 2, EDP.ReturnValue)
+        Friend ReadOnly RegexDataToken As RParams = RParams.DMS("data-token=""([^""]+)", 1, EDP.ReturnValue)
 #End Region
 #Region "Declarations M3U8"
        Friend ReadOnly Regex_M3U8_FilesList As RParams = RParams.DM("RESOLUTION=\d+x(\d+).*?[\r\n]*?(.+?m3u8.*)", 0, RegexReturn.List, EDP.ReturnValue)
--- a/SCrawler/API/PornHub/UserData.vb
+++ b/SCrawler/API/PornHub/UserData.vb
@@ -29,7 +29,7 @@ Namespace API.PornHub
        Private Const Name_DownloadFavorite As String = "DownloadFavorite"
        Private Const Name_DownloadGifs As String = "DownloadGifs"
        Private Const Name_DownloadPhotoOnlyFromModelHub As String = "DownloadPhotoOnlyFromModelHub"
-        Private Const Name_IsUser As String = "IsUser"
+        <Obsolete> Private Const Name_IsUser As String = "IsUser"
 #End Region
 #Region "Structures"
        Private Structure FlashVar : Implements IRegExCreator
@@ -117,6 +117,8 @@ Namespace API.PornHub
        Private Const PersonTypeUser As String = "users"
        Private Const PersonTypePornstar As String = "pornstar"
        Private Const PersonTypeCannel As String = "channels"
+        Private Const PersonTypePlaylist As String = "playlist"
+        Private Const PlaylistsLabelName As String = "Playlist"
 #End Region
 #Region "Person"
        Friend Property PersonType As String
@@ -133,7 +135,7 @@ Namespace API.PornHub
 #Region "Advanced fields"
        Friend Overrides ReadOnly Property FeedIsUser As Boolean
            Get
-                Return IsUser
+                Return IsUser Or SiteMode = SiteModes.Playlists
            End Get
        End Property
        Private Property PhotoPageModel As PhotoPageModels = PhotoPageModels.Undefined
@@ -144,12 +146,12 @@ Namespace API.PornHub
        Friend Property DownloadFavorite As Boolean = False
        Friend Property DownloadGifs As Boolean
        Friend Property DownloadPhotoOnlyFromModelHub As Boolean = True
-        Private _IsUser As Boolean = True
        Friend Overrides ReadOnly Property IsUser As Boolean
            Get
-                Return _IsUser
+                Return SiteMode = SiteModes.User
            End Get
        End Property
+        Friend Property SiteMode As SiteModes = SiteModes.User
        Friend Property QueryString As String
            Get
                If IsUser Then
@@ -164,7 +166,7 @@ Namespace API.PornHub
        End Property
        Friend Overrides ReadOnly Property SpecialLabels As IEnumerable(Of String)
            Get
-                Return {SearchRequestLabelName}
+                Return {SearchRequestLabelName, PlaylistsLabelName}
            End Get
        End Property
 #End Region
@@ -204,26 +206,32 @@ Namespace API.PornHub
 #Region "Loader"
        Private Function UpdateUserOptions(Optional ByVal Force As Boolean = False, Optional ByVal NewUrl As String = Nothing) As Boolean

-            If Not Force OrElse (Not IsUser AndAlso Not NewUrl.IsEmptyString AndAlso MyFileSettings.Exists) Then
+            If Not Force OrElse (Not IsUser AndAlso Not SiteMode = SiteModes.Playlists AndAlso Not NewUrl.IsEmptyString AndAlso MyFileSettings.Exists) Then
                Dim eObj As Plugin.ExchangeOptions = Nothing
                If Force Then eObj = MySettings.IsMyUser(NewUrl)
                If (Force And Not eObj.UserName.IsEmptyString) Or (Not Force And Not Name.IsEmptyString And NameTrue.IsEmptyString) Then
                    If Not If(Force, eObj.Options, Options).IsEmptyString Then
-                        If IsUser And Force Then
+                        If (IsUser Or SiteMode = SiteModes.Playlists) And Force Then
                            Return False
                        Else
-                            _IsUser = False
+                            SiteMode = SiteModes.Search
                            Options = If(Force, eObj.Options, Options)
+                            If Options.ToLower.StartsWith(PersonTypePlaylist) Then
+                                SiteMode = SiteModes.Playlists
+                                NameTrue = Options.ToLower.Replace(PersonTypePlaylist, String.Empty).StringTrim.TrimStart("/")
+                            Else
                                NameTrue = Options
+                            End If
                            If Not Force Then
-                                Settings.Labels.Add(SearchRequestLabelName)
-                                Labels.ListAddValue(SearchRequestLabelName, LNC)
+                                Dim l$ = IIf(SiteMode = SiteModes.Playlists, PlaylistsLabelName, SearchRequestLabelName)
+                                Settings.Labels.Add(l)
+                                Labels.ListAddValue(l, LNC)
                                Labels.Sort()
                                Return True
                            End If
                        End If
                    Else
-                        _IsUser = True
+                        SiteMode = SiteModes.User
                        Dim n$() = Name.Split("_")
                        If n.ListExists(2) Then
                            NameTrue = Name.Replace($"{n(0)}_", String.Empty)
@@ -247,7 +255,14 @@ Namespace API.PornHub
                    DownloadFavorite = .Value(Name_DownloadFavorite).FromXML(Of Boolean)(False)
                    DownloadGifs = .Value(Name_DownloadGifs).FromXML(Of Integer)(False)
                    DownloadPhotoOnlyFromModelHub = .Value(Name_DownloadPhotoOnlyFromModelHub).FromXML(Of Boolean)(True)
-                    _IsUser = .Value(Name_IsUser).FromXML(Of Boolean)(True)
+                    If .Contains(Name_SiteMode) Then
+                        SiteMode = .Value(Name_SiteMode).FromXML(Of Integer)(SiteModes.User)
+                    Else
+                        'TODELETE: PornHub 'IsUser' 20231113
+#Disable Warning BC40008
+                        SiteMode = IIf(.Value(Name_IsUser).FromXML(Of Boolean)(True), SiteModes.User, SiteModes.Search)
+#Enable Warning
+                    End If
                    UpdateUserOptions()
                Else
                    If UpdateUserOptions() Then .Value(Name_LabelsName) = LabelsString
@@ -261,7 +276,7 @@ Namespace API.PornHub
                    .Add(Name_DownloadFavorite, DownloadFavorite.BoolToInteger)
                    .Add(Name_DownloadGifs, DownloadGifs.BoolToInteger)
                    .Add(Name_DownloadPhotoOnlyFromModelHub, DownloadPhotoOnlyFromModelHub.BoolToInteger)
-                    .Add(Name_IsUser, IsUser.BoolToInteger)
+                    .Add(Name_SiteMode, CInt(SiteMode))

                    'Debug.WriteLine(GetNonUserUrl(0))
                    'Debug.WriteLine(GetNonUserUrl(2))
@@ -271,8 +286,11 @@ Namespace API.PornHub
 #End Region
 #Region "Downloading"
 #Region "Download override"
+        Private Const PlayListUrlPattern As String = "https://www.pornhub.com/playlist/viewChunked?id={0}&token={1}&page={2}"
+        Private PlaylistToken As String = String.Empty
        Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
            Try
+                PlaylistToken = String.Empty
                Responser.ResetStatus()

                If IsSavedPosts Then
@@ -283,7 +301,11 @@ Namespace API.PornHub
                Dim limit% = If(DownloadTopCount, -1)
                If DownloadVideos Then

-                    If IsSavedPosts Or Not IsUser Or PersonType = PersonTypeUser Then
+                    If SiteMode = SiteModes.Playlists Then
+                        Responser.Mode = Responser.Modes.Default
+                        GetPlaylistToken(Token)
+                        DownloadUserVideos(1, VideoTypes.Favorite, False, Token)
+                    ElseIf IsSavedPosts Or Not IsUser Or PersonType = PersonTypeUser Then
                        DownloadUserVideos(1, VideoTypes.Favorite, False, Token)
                    Else
                        If DownloadUploaded Then
@@ -365,6 +387,9 @@ Namespace API.PornHub
                        Throw New ArgumentException($"Type '{Type}' is not implemented in the video download function", "Type")
                    End If
                    If Page > 1 Then URL &= $"?page={Page}"
+                ElseIf SiteMode = SiteModes.Playlists Then
+                    If PlaylistToken.IsEmptyString Then Throw New ArgumentNullException("PlaylistToken", "Unable to get 'PlaylistToken'")
+                    URL = String.Format(PlayListUrlPattern, NameTrue, PlaylistToken, Page)
                Else
                    URL = GetNonUserUrl(Page)
                End If
@@ -375,7 +400,7 @@ Namespace API.PornHub
                Dim r$ = Responser.GetResponse(URL)
                If Not r.IsEmptyString Then
                    Dim l As List(Of UserVideo) = RegexFields(Of UserVideo)(r, {RegexUserVideos}, {6, 7, 3, 10})
-                    If l.ListExists Then l = l.ListTake(3, l.Count).ToList
+                    If l.ListExists And Not SiteMode = SiteModes.Playlists Then l = l.ListTake(3, l.Count).ToList
                    If l.ListExists Then
                        If IsUser Then
                            If Type = VideoTypes.Favorite Then
@@ -419,6 +444,15 @@ Namespace API.PornHub
                ProgressPre.Perform()
            End Try
        End Sub
+        Private Sub GetPlaylistToken(ByVal Token As CancellationToken)
+            Dim URL$ = GetNonUserUrl(0)
+            Try
+                Dim r$ = Responser.GetResponse(URL)
+                If Not r.IsEmptyString Then PlaylistToken = RegexReplace(r, RegexDataToken)
+            Catch ex As Exception
+                ProcessException(ex, Token, $"token getting error [{URL}]")
+            End Try
+        End Sub
 #End Region
 #Region "Download GIF"
        Private Sub DownloadUserGifs(ByVal Token As CancellationToken)
--- a/SCrawler/API/Reddit/UserData.vb
+++ b/SCrawler/API/Reddit/UserData.vb
@@ -23,9 +23,6 @@ Imports CView = SCrawler.API.Reddit.IRedditView.View
 Imports CPeriod = SCrawler.API.Reddit.IRedditView.Period
 Namespace API.Reddit
    Friend Class UserData : Inherits UserDataBase : Implements IChannelLimits, IRedditView
-#Region "XML names"
-        Private Const Name_TrueName As String = "TrueName"
-#End Region
 #Region "Declarations"
        Private Const CannelsLabelName As String = "Channels"
        Friend Const CannelsLabelName_ChannelsForm As String = "RChannels"
--- a/SCrawler/API/ThisVid/UserData.vb
+++ b/SCrawler/API/ThisVid/UserData.vb
@@ -20,9 +20,6 @@ Namespace API.ThisVid
        Private Const Name_DownloadPrivate As String = "DownloadPrivate"
        Private Const Name_DownloadFavourite As String = "DownloadFavourite"
        Private Const Name_DifferentFolders As String = "DifferentFolders"
-        Private Const Name_TrueName As String = "TrueName"
-        Private Const Name_SiteMode As String = "SiteMode"
-        Private Const Name_Arguments As String = "Arguments"
 #End Region
 #Region "Structures"
        Private Structure Album : Implements IRegExCreator
--- a/SCrawler/API/XVIDEOS/UserData.vb
+++ b/SCrawler/API/XVIDEOS/UserData.vb
@@ -17,9 +17,6 @@ Imports UTypes = SCrawler.API.Base.UserMedia.Types
 Namespace API.XVIDEOS
    Friend Class UserData : Inherits UserDataBase
 #Region "XML names"
-        Private Const Name_SiteMode As String = "SiteMode"
-        Private Const Name_TrueName As String = "TrueName"
-        Private Const Name_Arguments As String = "Arguments"
        Private Const Name_PersonType As String = "PersonType"
 #End Region
 #Region "Structures"
--- a/SCrawler/API/Xhamster/UserData.vb
+++ b/SCrawler/API/Xhamster/UserData.vb
@@ -18,10 +18,7 @@ Imports UTypes = SCrawler.API.Base.UserMedia.Types
 Namespace API.Xhamster
    Friend Class UserData : Inherits UserDataBase
 #Region "XML names"
-        Private Const Name_TrueName As String = "TrueName"
        Private Const Name_Gender As String = "Gender"
-        Private Const Name_SiteMode As String = "SiteMode"
-        Private Const Name_Arguments As String = "Arguments"
 #End Region
 #Region "Declarations"
        Friend Overrides ReadOnly Property FeedIsUser As Boolean