diff --git a/SCrawler/API/Base/Structures.vb b/SCrawler/API/Base/Structures.vb index 51abe75..e8fdb6e 100644 --- a/SCrawler/API/Base/Structures.vb +++ b/SCrawler/API/Base/Structures.vb @@ -18,6 +18,7 @@ Namespace API.Base Tags = 2 Categories = 3 Pornstars = 4 + Playlists = 5 End Enum Friend Structure UserMedia : Implements IUserMedia, IEquatable(Of UserMedia), IEContainerProvider #Region "XML Names" diff --git a/SCrawler/API/Base/UserDataBase.vb b/SCrawler/API/Base/UserDataBase.vb index 75590fc..95f56b0 100644 --- a/SCrawler/API/Base/UserDataBase.vb +++ b/SCrawler/API/Base/UserDataBase.vb @@ -169,6 +169,11 @@ Namespace API.Base Protected Const Name_UseMD5Comparison As String = "UseMD5Comparison" Protected Const Name_RemoveExistingDuplicates As String = "RemoveExistingDuplicates" Protected Const Name_StartMD5Checked As String = "StartMD5Checked" +#Region "Additional names" + Protected Const Name_SiteMode As String = "SiteMode" + Protected Const Name_TrueName As String = "TrueName" + Protected Const Name_Arguments As String = "Arguments" +#End Region #End Region #Region "Declarations" #Region "Host, Site, Progress" diff --git a/SCrawler/API/Mastodon/UserData.vb b/SCrawler/API/Mastodon/UserData.vb index 3c28a3f..9beeedd 100644 --- a/SCrawler/API/Mastodon/UserData.vb +++ b/SCrawler/API/Mastodon/UserData.vb @@ -18,7 +18,6 @@ Namespace API.Mastodon Friend Class UserData : Inherits Twitter.UserData #Region "XML names" Private Const Name_UserDomain As String = "UserDomain" - Private Const Name_TrueName As String = "TrueName" #End Region #Region "Declarations" Private _UserDomain As String = String.Empty diff --git a/SCrawler/API/PornHub/Declarations.vb b/SCrawler/API/PornHub/Declarations.vb index 8909845..0dddd96 100644 --- a/SCrawler/API/PornHub/Declarations.vb +++ b/SCrawler/API/PornHub/Declarations.vb @@ -24,6 +24,7 @@ Namespace API.PornHub 0, RegexOptions.Singleline, RegexReturn.List, EDP.ReturnValue, UnicodeHexConverter) Friend ReadOnly RegexVideo_Video_VideoKey As RParams = RParams.DMS("viewkey=([\w\d]+)", 1, EDP.ReturnValue) Friend ReadOnly RegexVideoPageTitle As RParams = RParams.DMS("meta (property|name)=""[^:]+?:title"" content=""([^""]+)""", 2, EDP.ReturnValue) + Friend ReadOnly RegexDataToken As RParams = RParams.DMS("data-token=""([^""]+)", 1, EDP.ReturnValue) #End Region #Region "Declarations M3U8" Friend ReadOnly Regex_M3U8_FilesList As RParams = RParams.DM("RESOLUTION=\d+x(\d+).*?[\r\n]*?(.+?m3u8.*)", 0, RegexReturn.List, EDP.ReturnValue) diff --git a/SCrawler/API/PornHub/UserData.vb b/SCrawler/API/PornHub/UserData.vb index d9a8606..b303231 100644 --- a/SCrawler/API/PornHub/UserData.vb +++ b/SCrawler/API/PornHub/UserData.vb @@ -29,7 +29,7 @@ Namespace API.PornHub Private Const Name_DownloadFavorite As String = "DownloadFavorite" Private Const Name_DownloadGifs As String = "DownloadGifs" Private Const Name_DownloadPhotoOnlyFromModelHub As String = "DownloadPhotoOnlyFromModelHub" - Private Const Name_IsUser As String = "IsUser" + Private Const Name_IsUser As String = "IsUser" #End Region #Region "Structures" Private Structure FlashVar : Implements IRegExCreator @@ -117,6 +117,8 @@ Namespace API.PornHub Private Const PersonTypeUser As String = "users" Private Const PersonTypePornstar As String = "pornstar" Private Const PersonTypeCannel As String = "channels" + Private Const PersonTypePlaylist As String = "playlist" + Private Const PlaylistsLabelName As String = "Playlist" #End Region #Region "Person" Friend Property PersonType As String @@ -133,7 +135,7 @@ Namespace API.PornHub #Region "Advanced fields" Friend Overrides ReadOnly Property FeedIsUser As Boolean Get - Return IsUser + Return IsUser Or SiteMode = SiteModes.Playlists End Get End Property Private Property PhotoPageModel As PhotoPageModels = PhotoPageModels.Undefined @@ -144,12 +146,12 @@ Namespace API.PornHub Friend Property DownloadFavorite As Boolean = False Friend Property DownloadGifs As Boolean Friend Property DownloadPhotoOnlyFromModelHub As Boolean = True - Private _IsUser As Boolean = True Friend Overrides ReadOnly Property IsUser As Boolean Get - Return _IsUser + Return SiteMode = SiteModes.User End Get End Property + Friend Property SiteMode As SiteModes = SiteModes.User Friend Property QueryString As String Get If IsUser Then @@ -164,7 +166,7 @@ Namespace API.PornHub End Property Friend Overrides ReadOnly Property SpecialLabels As IEnumerable(Of String) Get - Return {SearchRequestLabelName} + Return {SearchRequestLabelName, PlaylistsLabelName} End Get End Property #End Region @@ -204,26 +206,32 @@ Namespace API.PornHub #Region "Loader" Private Function UpdateUserOptions(Optional ByVal Force As Boolean = False, Optional ByVal NewUrl As String = Nothing) As Boolean - If Not Force OrElse (Not IsUser AndAlso Not NewUrl.IsEmptyString AndAlso MyFileSettings.Exists) Then + If Not Force OrElse (Not IsUser AndAlso Not SiteMode = SiteModes.Playlists AndAlso Not NewUrl.IsEmptyString AndAlso MyFileSettings.Exists) Then Dim eObj As Plugin.ExchangeOptions = Nothing If Force Then eObj = MySettings.IsMyUser(NewUrl) If (Force And Not eObj.UserName.IsEmptyString) Or (Not Force And Not Name.IsEmptyString And NameTrue.IsEmptyString) Then If Not If(Force, eObj.Options, Options).IsEmptyString Then - If IsUser And Force Then + If (IsUser Or SiteMode = SiteModes.Playlists) And Force Then Return False Else - _IsUser = False + SiteMode = SiteModes.Search Options = If(Force, eObj.Options, Options) - NameTrue = Options + If Options.ToLower.StartsWith(PersonTypePlaylist) Then + SiteMode = SiteModes.Playlists + NameTrue = Options.ToLower.Replace(PersonTypePlaylist, String.Empty).StringTrim.TrimStart("/") + Else + NameTrue = Options + End If If Not Force Then - Settings.Labels.Add(SearchRequestLabelName) - Labels.ListAddValue(SearchRequestLabelName, LNC) + Dim l$ = IIf(SiteMode = SiteModes.Playlists, PlaylistsLabelName, SearchRequestLabelName) + Settings.Labels.Add(l) + Labels.ListAddValue(l, LNC) Labels.Sort() Return True End If End If Else - _IsUser = True + SiteMode = SiteModes.User Dim n$() = Name.Split("_") If n.ListExists(2) Then NameTrue = Name.Replace($"{n(0)}_", String.Empty) @@ -247,7 +255,14 @@ Namespace API.PornHub DownloadFavorite = .Value(Name_DownloadFavorite).FromXML(Of Boolean)(False) DownloadGifs = .Value(Name_DownloadGifs).FromXML(Of Integer)(False) DownloadPhotoOnlyFromModelHub = .Value(Name_DownloadPhotoOnlyFromModelHub).FromXML(Of Boolean)(True) - _IsUser = .Value(Name_IsUser).FromXML(Of Boolean)(True) + If .Contains(Name_SiteMode) Then + SiteMode = .Value(Name_SiteMode).FromXML(Of Integer)(SiteModes.User) + Else + 'TODELETE: PornHub 'IsUser' 20231113 +#Disable Warning BC40008 + SiteMode = IIf(.Value(Name_IsUser).FromXML(Of Boolean)(True), SiteModes.User, SiteModes.Search) +#Enable Warning + End If UpdateUserOptions() Else If UpdateUserOptions() Then .Value(Name_LabelsName) = LabelsString @@ -261,7 +276,7 @@ Namespace API.PornHub .Add(Name_DownloadFavorite, DownloadFavorite.BoolToInteger) .Add(Name_DownloadGifs, DownloadGifs.BoolToInteger) .Add(Name_DownloadPhotoOnlyFromModelHub, DownloadPhotoOnlyFromModelHub.BoolToInteger) - .Add(Name_IsUser, IsUser.BoolToInteger) + .Add(Name_SiteMode, CInt(SiteMode)) 'Debug.WriteLine(GetNonUserUrl(0)) 'Debug.WriteLine(GetNonUserUrl(2)) @@ -271,8 +286,11 @@ Namespace API.PornHub #End Region #Region "Downloading" #Region "Download override" + Private Const PlayListUrlPattern As String = "https://www.pornhub.com/playlist/viewChunked?id={0}&token={1}&page={2}" + Private PlaylistToken As String = String.Empty Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken) Try + PlaylistToken = String.Empty Responser.ResetStatus() If IsSavedPosts Then @@ -283,7 +301,11 @@ Namespace API.PornHub Dim limit% = If(DownloadTopCount, -1) If DownloadVideos Then - If IsSavedPosts Or Not IsUser Or PersonType = PersonTypeUser Then + If SiteMode = SiteModes.Playlists Then + Responser.Mode = Responser.Modes.Default + GetPlaylistToken(Token) + DownloadUserVideos(1, VideoTypes.Favorite, False, Token) + ElseIf IsSavedPosts Or Not IsUser Or PersonType = PersonTypeUser Then DownloadUserVideos(1, VideoTypes.Favorite, False, Token) Else If DownloadUploaded Then @@ -365,6 +387,9 @@ Namespace API.PornHub Throw New ArgumentException($"Type '{Type}' is not implemented in the video download function", "Type") End If If Page > 1 Then URL &= $"?page={Page}" + ElseIf SiteMode = SiteModes.Playlists Then + If PlaylistToken.IsEmptyString Then Throw New ArgumentNullException("PlaylistToken", "Unable to get 'PlaylistToken'") + URL = String.Format(PlayListUrlPattern, NameTrue, PlaylistToken, Page) Else URL = GetNonUserUrl(Page) End If @@ -375,7 +400,7 @@ Namespace API.PornHub Dim r$ = Responser.GetResponse(URL) If Not r.IsEmptyString Then Dim l As List(Of UserVideo) = RegexFields(Of UserVideo)(r, {RegexUserVideos}, {6, 7, 3, 10}) - If l.ListExists Then l = l.ListTake(3, l.Count).ToList + If l.ListExists And Not SiteMode = SiteModes.Playlists Then l = l.ListTake(3, l.Count).ToList If l.ListExists Then If IsUser Then If Type = VideoTypes.Favorite Then @@ -419,6 +444,15 @@ Namespace API.PornHub ProgressPre.Perform() End Try End Sub + Private Sub GetPlaylistToken(ByVal Token As CancellationToken) + Dim URL$ = GetNonUserUrl(0) + Try + Dim r$ = Responser.GetResponse(URL) + If Not r.IsEmptyString Then PlaylistToken = RegexReplace(r, RegexDataToken) + Catch ex As Exception + ProcessException(ex, Token, $"token getting error [{URL}]") + End Try + End Sub #End Region #Region "Download GIF" Private Sub DownloadUserGifs(ByVal Token As CancellationToken) diff --git a/SCrawler/API/Reddit/UserData.vb b/SCrawler/API/Reddit/UserData.vb index 2ca625a..1d1f481 100644 --- a/SCrawler/API/Reddit/UserData.vb +++ b/SCrawler/API/Reddit/UserData.vb @@ -23,9 +23,6 @@ Imports CView = SCrawler.API.Reddit.IRedditView.View Imports CPeriod = SCrawler.API.Reddit.IRedditView.Period Namespace API.Reddit Friend Class UserData : Inherits UserDataBase : Implements IChannelLimits, IRedditView -#Region "XML names" - Private Const Name_TrueName As String = "TrueName" -#End Region #Region "Declarations" Private Const CannelsLabelName As String = "Channels" Friend Const CannelsLabelName_ChannelsForm As String = "RChannels" diff --git a/SCrawler/API/ThisVid/UserData.vb b/SCrawler/API/ThisVid/UserData.vb index 88ba6e9..e0adb36 100644 --- a/SCrawler/API/ThisVid/UserData.vb +++ b/SCrawler/API/ThisVid/UserData.vb @@ -20,9 +20,6 @@ Namespace API.ThisVid Private Const Name_DownloadPrivate As String = "DownloadPrivate" Private Const Name_DownloadFavourite As String = "DownloadFavourite" Private Const Name_DifferentFolders As String = "DifferentFolders" - Private Const Name_TrueName As String = "TrueName" - Private Const Name_SiteMode As String = "SiteMode" - Private Const Name_Arguments As String = "Arguments" #End Region #Region "Structures" Private Structure Album : Implements IRegExCreator diff --git a/SCrawler/API/XVIDEOS/UserData.vb b/SCrawler/API/XVIDEOS/UserData.vb index 24f3248..b60f2d2 100644 --- a/SCrawler/API/XVIDEOS/UserData.vb +++ b/SCrawler/API/XVIDEOS/UserData.vb @@ -17,9 +17,6 @@ Imports UTypes = SCrawler.API.Base.UserMedia.Types Namespace API.XVIDEOS Friend Class UserData : Inherits UserDataBase #Region "XML names" - Private Const Name_SiteMode As String = "SiteMode" - Private Const Name_TrueName As String = "TrueName" - Private Const Name_Arguments As String = "Arguments" Private Const Name_PersonType As String = "PersonType" #End Region #Region "Structures" diff --git a/SCrawler/API/Xhamster/UserData.vb b/SCrawler/API/Xhamster/UserData.vb index 0987d03..0131fcc 100644 --- a/SCrawler/API/Xhamster/UserData.vb +++ b/SCrawler/API/Xhamster/UserData.vb @@ -18,10 +18,7 @@ Imports UTypes = SCrawler.API.Base.UserMedia.Types Namespace API.Xhamster Friend Class UserData : Inherits UserDataBase #Region "XML names" - Private Const Name_TrueName As String = "TrueName" Private Const Name_Gender As String = "Gender" - Private Const Name_SiteMode As String = "SiteMode" - Private Const Name_Arguments As String = "Arguments" #End Region #Region "Declarations" Friend Overrides ReadOnly Property FeedIsUser As Boolean