geonu/SCrawler
1
0
Fork 0
mirror of https://github.com/AAndyProgram/SCrawler.git synced 2026-03-14 15:52:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

2025.9.1.0

Browse Source 
API.PornHub: update regex and data parsing
This commit is contained in:
Andy
2025-09-01 16:41:44 +03:00
parent 5857fcfae3
commit 1404afdfa3
4 changed files with 32 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
Changelog.md
View File

@@ -1,3 +1,10 @@
# 2025.9.1.0
*2025-09-01*
- Fixed
- PornHub: data is not downloading
# 2025.8.30.0 # 2025.8.30.0
*2025-08-30* *2025-08-30*

2
SCrawler/API/PornHub/Declarations.vb
View File

@@ -20,7 +20,7 @@ Namespace API.PornHub
Friend ReadOnly RegexVideo_FlashVars_Vars As RParams = RParams.DM("var ([\w\d]{10,})=("".+?)(?=(;|\Z))", 0, RegexReturn.List) Friend ReadOnly RegexVideo_FlashVars_Vars As RParams = RParams.DM("var ([\w\d]{10,})=("".+?)(?=(;|\Z))", 0, RegexReturn.List)
Friend ReadOnly RegexVideo_FlashVars_Compiler As RParams = RParams.DM("(?<=\*/)([\w\d\S]{10,})", 0, RegexReturn.List) Friend ReadOnly RegexVideo_FlashVars_Compiler As RParams = RParams.DM("(?<=\*/)([\w\d\S]{10,})", 0, RegexReturn.List)
Friend ReadOnly RegexVideo_FlashVars_UrlResolution As RParams = RParams.DMS("/(\d+)[^/]+\.mp4", 1, EDP.ReturnValue) Friend ReadOnly RegexVideo_FlashVars_UrlResolution As RParams = RParams.DMS("/(\d+)[^/]+\.mp4", 1, EDP.ReturnValue)
Friend ReadOnly RegexUserVideos As RParams = RParams.DM("(\<li class=""pcVideoListItem)((?:(?!/li\>).)*?)(\<div.class=.private-vid-title((?:(?!/li\>).)*?)|)(\<a.href=.([^""]+?)"".title=.([^""]*?)"")(((?:(?!/li\>).)+?)(\<div class=.videoUploaderBlock)|)((?:(?!/li\>).)*?)(\</li\>)", Friend ReadOnly RegexUserVideos As RParams = RParams.DM("(\<li class=""pcVideoListItem)((?:(?!/li\>).)*?)(\<div.class=.private-vid-title((?:(?!/li\>).)*?)|)(\<a.href=.([^""]+?)"".title=.([^""]*?)"")(((?:(?!/li\>).)+?)(\<div class=.videoUploaderBlock.*?href=""([^""]+)"")|)((?:(?!/li\>).)*?)(\</li\>)",
0, RegexOptions.Singleline, RegexReturn.List, EDP.ReturnValue, UnicodeHexConverter) 0, RegexOptions.Singleline, RegexReturn.List, EDP.ReturnValue, UnicodeHexConverter)
Friend ReadOnly RegexVideo_Video_VideoKey As RParams = RParams.DMS("viewkey=([\w\d]+)", 1, EDP.ReturnValue) Friend ReadOnly RegexVideo_Video_VideoKey As RParams = RParams.DMS("viewkey=([\w\d]+)", 1, EDP.ReturnValue)
Friend ReadOnly RegexVideoPageTitle As RParams = RParams.DMS("meta (property|name)=""[^:]+?:title"" content=""([^""]+)""", 2, EDP.ReturnValue) Friend ReadOnly RegexVideoPageTitle As RParams = RParams.DMS("meta (property|name)=""[^:]+?:title"" content=""([^""]+)""", 2, EDP.ReturnValue)

29
SCrawler/API/PornHub/UserData.vb
View File

@@ -50,6 +50,7 @@ Namespace API.PornHub
Friend URL As String Friend URL As String
Friend ID As String Friend ID As String
Friend Title As String Friend Title As String
Friend UserRef As String
Friend Type As VideoTypes Friend Type As VideoTypes
Friend Function ToUserMedia(Optional ByVal SpecialFolder As String = Nothing) As UserMedia Friend Function ToUserMedia(Optional ByVal SpecialFolder As String = Nothing) As UserMedia
Return New UserMedia(URL, UTypes.VideoPre) With { Return New UserMedia(URL, UTypes.VideoPre) With {
@@ -66,14 +67,16 @@ Namespace API.PornHub
URL = String.Empty URL = String.Empty
Else Else
URL = String.Format(UrlPattern, URL.TrimStart("/")) URL = String.Format(UrlPattern, URL.TrimStart("/"))
Title = TitleHtmlConverter(ParamsArray(1)) Title = TitleHtmlConverter(ParamsArray(1))
If Not ParamsArray(2).IsEmptyString Then If Not ParamsArray(2).IsEmptyString Then
Type = VideoTypes.Private Type = VideoTypes.Private
ElseIf Not ParamsArray(3).IsEmptyString Then 'ElseIf Not ParamsArray(3).IsEmptyString Then
Type = VideoTypes.Tagged ' Type = VideoTypes.Tagged
Else Else
Type = VideoTypes.Uploaded Type = VideoTypes.Uploaded
End If End If
If Not ParamsArray(3).IsEmptyString Then UserRef = ParamsArray(3).StringTrim
End If End If
End If End If
Return Me Return Me
@@ -338,10 +341,13 @@ Namespace API.PornHub
Dim tryNextPage As Boolean = False Dim tryNextPage As Boolean = False
Dim limit% = If(DownloadTopCount, -1) Dim limit% = If(DownloadTopCount, -1)
Dim cBefore% = _TempMediaList.Count Dim cBefore% = _TempMediaList.Count
Dim usrRef$ = String.Empty
Dim npd$ = "?"
If IsUser Then If IsUser Then
URL = $"https://www.pornhub.com/{PersonType}/{NameTrue}" URL = $"https://www.pornhub.com/{PersonType}/{NameTrue}"
usrRef = $"/{PersonType}/{NameTrue}"
If Type = VideoTypes.Uploaded Then If Type = VideoTypes.Uploaded Then
URL &= "/videos/upload" If Not PersonType = PersonTypeCannel Then URL &= "/videos/upload?o=mr" : npd = "&"
ElseIf Type = VideoTypes.Tagged Then ElseIf Type = VideoTypes.Tagged Then
If Not SecondMode Then URL &= "/videos" If Not SecondMode Then URL &= "/videos"
specFolder = "Tagged" specFolder = "Tagged"
@@ -354,7 +360,7 @@ Namespace API.PornHub
Else Else
Throw New ArgumentException($"Type '{Type}' is not implemented in the video download function", "Type") Throw New ArgumentException($"Type '{Type}' is not implemented in the video download function", "Type")
End If End If
If Page > 1 Then URL &= $"?page={Page}" If Page > 1 Then URL &= $"{npd}page={Page}"
ElseIf SiteMode = SiteModes.Playlists Then ElseIf SiteMode = SiteModes.Playlists Then
If PlaylistToken.IsEmptyString Then Throw New ArgumentNullException("PlaylistToken", "Unable to get 'PlaylistToken'") If PlaylistToken.IsEmptyString Then Throw New ArgumentNullException("PlaylistToken", "Unable to get 'PlaylistToken'")
URL = String.Format(PlayListUrlPattern, NameTrue, PlaylistToken, Page) URL = String.Format(PlayListUrlPattern, NameTrue, PlaylistToken, Page)
@@ -367,12 +373,21 @@ Namespace API.PornHub
'Debug.WriteLine(URL) 'Debug.WriteLine(URL)
Dim r$ = Responser.GetResponse(URL) Dim r$ = Responser.GetResponse(URL)
If Not r.IsEmptyString Then If Not r.IsEmptyString Then
Dim l As List(Of UserVideo) = RegexFields(Of UserVideo)(r, {RegexUserVideos}, {6, 7, 3, 10}) Dim l As List(Of UserVideo) = RegexFields(Of UserVideo)(r, {RegexUserVideos}, {6, 7, 3, 11})
'If l.ListExists And Not SiteMode = SiteModes.Playlists Then l = l.ListTake(3, l.Count).ToList 'If l.ListExists And Not SiteMode = SiteModes.Playlists Then l = l.ListTake(3, l.Count).ToList
If l.ListExists And Not SiteMode = SiteModes.Playlists Then l = l.ListTake(1, l.Count).ToList If l.ListExists And Not SiteMode = SiteModes.Playlists And Not IsUser Then l = l.ListTake(1, l.Count).ToList
If l.ListExists Then If l.ListExists Then
If IsUser Then If IsUser Then
If Type = VideoTypes.Favorite Then If Type = VideoTypes.Tagged Then
l = l.ListTake(4, l.Count)
If l.ListExists Then l.RemoveAll(Function(uv) Not uv.UserRef.IsEmptyString AndAlso uv.UserRef = usrRef)
ElseIf Type = VideoTypes.Uploaded Then
If PersonType = PersonTypeCannel Then
l = l.ListTake(4, l.Count)
Else
l.RemoveAll(Function(uv) uv.UserRef.IsEmptyString OrElse Not uv.UserRef = usrRef)
End If
ElseIf Type = VideoTypes.Favorite Then
l.RemoveAll(Function(uv) uv.Type = VideoTypes.Private) l.RemoveAll(Function(uv) uv.Type = VideoTypes.Private)
ElseIf Not PersonType = PersonTypeCannel Then ElseIf Not PersonType = PersonTypeCannel Then
l.RemoveAll(Function(uv) Not uv.Type = Type) l.RemoveAll(Function(uv) Not uv.Type = Type)

4
SCrawler/My Project/AssemblyInfo.vb
View File

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
' by using the '*' as shown below: ' by using the '*' as shown below:
' <Assembly: AssemblyVersion("1.0.*")> ' <Assembly: AssemblyVersion("1.0.*")>
<Assembly: AssemblyVersion("2025.8.30.0")> <Assembly: AssemblyVersion("2025.9.1.0")>
<Assembly: AssemblyFileVersion("2025.8.30.0")> <Assembly: AssemblyFileVersion("2025.9.1.0")>
<Assembly: NeutralResourcesLanguage("en")> <Assembly: NeutralResourcesLanguage("en")>