From 2bacc17ac48a03f130ab583c04a6232de0dcc7a1 Mon Sep 17 00:00:00 2001 From: Andy <88590076+AAndyProgram@users.noreply.github.com> Date: Sun, 17 Mar 2024 23:02:00 +0300 Subject: [PATCH] 2024.3.17.0 API.OnlyFans: add stories download API.PornHub: fix list trim --- SCrawler/API/OnlyFans/SiteSettings.vb | 10 ++- SCrawler/API/OnlyFans/UserData.vb | 81 ++++++++++++++++++-- SCrawler/API/OnlyFans/UserExchangeOptions.vb | 8 ++ SCrawler/API/PornHub/UserData.vb | 4 +- SCrawler/Download/Feed/FeedMedia.vb | 2 +- 5 files changed, 94 insertions(+), 11 deletions(-) diff --git a/SCrawler/API/OnlyFans/SiteSettings.vb b/SCrawler/API/OnlyFans/SiteSettings.vb index 39d4581..8bfe184 100644 --- a/SCrawler/API/OnlyFans/SiteSettings.vb +++ b/SCrawler/API/OnlyFans/SiteSettings.vb @@ -18,10 +18,14 @@ Namespace API.OnlyFans Friend Class SiteSettings : Inherits SiteSettingsBase #Region "Declarations" #Region "Options" + + Friend ReadOnly Property DownloadTimeline As PropertyValue + + Friend ReadOnly Property DownloadStories As PropertyValue - Friend Property DownloadHighlights As PropertyValue + Friend ReadOnly Property DownloadHighlights As PropertyValue - Friend Property DownloadChatMedia As PropertyValue + Friend ReadOnly Property DownloadChatMedia As PropertyValue #End Region #Region "Headers" Private Const HeaderBrowser As String = "sec-ch-ua" @@ -145,6 +149,8 @@ Namespace API.OnlyFans UserAgent = New PropertyValue(IIf(.UserAgentExists, .UserAgent, String.Empty), GetType(String), Sub(v) UpdateHeader(NameOf(UserAgent), v)) End With + DownloadTimeline = New PropertyValue(True) + DownloadStories = New PropertyValue(True) DownloadHighlights = New PropertyValue(True) DownloadChatMedia = New PropertyValue(True) diff --git a/SCrawler/API/OnlyFans/UserData.vb b/SCrawler/API/OnlyFans/UserData.vb index 00b8a84..61f0602 100644 --- a/SCrawler/API/OnlyFans/UserData.vb +++ b/SCrawler/API/OnlyFans/UserData.vb @@ -22,6 +22,8 @@ Imports UStates = SCrawler.API.Base.UserMedia.States Namespace API.OnlyFans Friend Class UserData : Inherits UserDataBase #Region "XML names" + Private Const Name_MediaDownloadTimeline As String = "MediaDownloadTimeline" + Private Const Name_MediaDownloadStories As String = "MediaDownloadStories" Private Const Name_MediaDownloadHighlights As String = "DownloadHighlights" Private Const Name_MediaDownloadChatMedia As String = "DownloadChatMedia" #End Region @@ -30,6 +32,8 @@ Namespace API.OnlyFans Private Const HeaderSign As String = "Sign" Private Const HeaderTime As String = "Time" Private ReadOnly HighlightsList As List(Of String) + Friend Property MediaDownloadTimeline As Boolean = True + Friend Property MediaDownloadStories As Boolean = True Friend Property MediaDownloadHighlights As Boolean = True Friend Property MediaDownloadChatMedia As Boolean = True Private ReadOnly Property MySettings As SiteSettings @@ -42,9 +46,13 @@ Namespace API.OnlyFans Protected Overrides Sub LoadUserInformation_OptionalFields(ByRef Container As XmlFile, ByVal Loading As Boolean) With Container If Loading Then + MediaDownloadTimeline = .Value(Name_MediaDownloadTimeline).FromXML(Of Boolean)(True) + MediaDownloadStories = .Value(Name_MediaDownloadStories).FromXML(Of Boolean)(True) MediaDownloadHighlights = .Value(Name_MediaDownloadHighlights).FromXML(Of Boolean)(True) MediaDownloadChatMedia = .Value(Name_MediaDownloadChatMedia).FromXML(Of Boolean)(True) Else + .Add(Name_MediaDownloadTimeline, MediaDownloadTimeline.BoolToInteger) + .Add(Name_MediaDownloadStories, MediaDownloadStories.BoolToInteger) .Add(Name_MediaDownloadHighlights, MediaDownloadHighlights.BoolToInteger) .Add(Name_MediaDownloadChatMedia, MediaDownloadChatMedia.BoolToInteger) End If @@ -58,6 +66,8 @@ Namespace API.OnlyFans Friend Overrides Sub ExchangeOptionsSet(ByVal Obj As Object) If Not Obj Is Nothing AndAlso TypeOf Obj Is UserExchangeOptions Then With DirectCast(Obj, UserExchangeOptions) + MediaDownloadTimeline = .DownloadTimeline + MediaDownloadStories = .DownloadStories MediaDownloadHighlights = .DownloadHighlights MediaDownloadChatMedia = .DownloadChatMedia End With @@ -90,8 +100,15 @@ Namespace API.OnlyFans Responser.Cookies.Clear() AddHandler Responser.ResponseReceived, AddressOf Responser_ResponseReceived UpdateCookieHeader() - DownloadTimeline(IIf(IsSavedPosts, 0, String.Empty), Token) + If Not IsSavedPosts Then + If ID.IsEmptyString Then GetUserID() + If ID.IsEmptyString Then Throw New ArgumentNullException("ID", "Unable to get user ID") + End If + + If MediaDownloadTimeline Then DownloadTimeline(IIf(IsSavedPosts, 0, String.Empty), Token) + If Not IsSavedPosts Then + If MediaDownloadStories And FunctionErr = FunctionErrDef Then DownloadStories(Token) If MediaDownloadHighlights And FunctionErr = FunctionErrDef Then DownloadHighlights(Token) If MediaDownloadChatMedia And FunctionErr = FunctionErrDef Then DownloadChatMedia(0, Token) End If @@ -114,6 +131,7 @@ Namespace API.OnlyFans Return ErrValue <> 2 End Function Friend Const A_HIGHLIGHT As String = "HL" + Friend Const A_STORIES As String = "ST" Friend Const A_MESSAGE As String = "MSG" Private Const BaseUrlPattern As String = "https://onlyfans.com{0}" #Region "Download timeline" @@ -133,9 +151,6 @@ Namespace API.OnlyFans If IsSavedPosts Then path = $"/api2/v2/posts/bookmarks/all/?format=infinite&limit=10&offset={Cursor}" Else - If ID.IsEmptyString Then GetUserID() - If ID.IsEmptyString Then Throw New ArgumentNullException("ID", "Unable to get user ID") - path = $"/api2/v2/users/{ID}/posts/medias?limit=50&order=publish_date_desc&skip_users=all&format=infinite&counters=1" If Not Cursor.IsEmptyString Then path &= $"&counters=0&beforePublishTime={Cursor}" Else path &= "&counters=1" End If @@ -250,7 +265,7 @@ Namespace API.OnlyFans If Not r.IsEmptyString Then Using j As EContainer = JsonDocument.Parse(r) If j.ListExists Then - specFolder = j.Value("title").StringRemoveWinForbiddenSymbols.IfNullOrEmpty(HLID) + specFolder = "Highlights\" & j.Value("title").StringRemoveWinForbiddenSymbols.IfNullOrEmpty(HLID) specFolder &= "*" With j("stories") If .ListExists Then @@ -280,6 +295,57 @@ Namespace API.OnlyFans Loop While Not _complete End Sub #End Region +#Region "Download stories" + Private Sub DownloadStories(ByVal Token As CancellationToken) + Dim url$ = String.Empty + Dim _complete As Boolean = True + Do + Try + Dim specFolder$ = "Stories" + Dim postID$, postDate$ + Dim media As List(Of UserMedia) + Dim result As Boolean + Dim path$ = $"/api2/v2/users/{ID}/stories" + If UpdateSignature(path) Then + url = String.Format(BaseUrlPattern, path) + ThrowAny(Token) + Dim r$ = Responser.GetResponse(url) + If Not r.IsEmptyString Then + Using j As EContainer = JsonDocument.Parse(r) + If j.ListExists Then + ProgressPre.ChangeMax(j.Count) + For Each n As EContainer In j + ProgressPre.Perform() + With n.ItemF({"media", 0}) + If .ListExists Then + postID = $"{A_STORIES}_{ .Value("id")}" + postDate = .Value("createdAt") + Else + postID = String.Empty + postDate = String.Empty + End If + End With + If Not postID.IsEmptyString Then + If Not _TempPostsList.Contains(postID) Then + _TempPostsList.Add(postID) + Else + Exit Sub + End If + End If + result = False + media = TryCreateMedia(n, postID, postDate, result, True, specFolder,, False) + If result Then _TempMediaList.ListAddList(media, LNC) + Next + End If + End Using + End If + End If + Catch ex As Exception + _complete = ProcessFunctionErrComplete(ProcessException(ex, Token, $"stories downloading error [{url}]")) + End Try + Loop While Not _complete + End Sub +#End Region #Region "Download chat media" Private Sub DownloadChatMedia(ByVal Cursor As Integer, ByVal Token As CancellationToken) Dim url$ = String.Empty @@ -329,7 +395,8 @@ Namespace API.OnlyFans #End Region Private Function TryCreateMedia(ByVal n As EContainer, ByVal PostID As String, Optional ByVal PostDate As String = Nothing, Optional ByRef Result As Boolean = False, Optional ByVal IsHL As Boolean = False, - Optional ByVal SpecFolder As String = Nothing, Optional ByVal PostUserID As String = Nothing) As List(Of UserMedia) + Optional ByVal SpecFolder As String = Nothing, Optional ByVal PostUserID As String = Nothing, + Optional ByVal TryUseOFS As Boolean = True) As List(Of UserMedia) Dim postUrl$, postUrlBase$, ext$ Dim t As UTypes Dim mList As New List(Of UserMedia) @@ -348,7 +415,7 @@ Namespace API.OnlyFans Case "video" t = UTypes.Video ext = "mp4" - If postUrl.IsEmptyString And Not IsHL Then + If postUrl.IsEmptyString And Not IsHL And TryUseOFS Then t = UTypes.VideoPre _AbsMediaIndex += 1 If Not PostUserID.IsEmptyString And IsSingleObjectDownload Then _ diff --git a/SCrawler/API/OnlyFans/UserExchangeOptions.vb b/SCrawler/API/OnlyFans/UserExchangeOptions.vb index dfe337d..0a8752d 100644 --- a/SCrawler/API/OnlyFans/UserExchangeOptions.vb +++ b/SCrawler/API/OnlyFans/UserExchangeOptions.vb @@ -9,17 +9,25 @@ Imports SCrawler.Plugin.Attributes Namespace API.OnlyFans Friend Class UserExchangeOptions + + Friend Property DownloadTimeline As Boolean + + Friend Property DownloadStories As Boolean Friend Property DownloadHighlights As Boolean Friend Property DownloadChatMedia As Boolean Private ReadOnly MySettings As SiteSettings Friend Sub New(ByVal u As UserData) + DownloadTimeline = u.MediaDownloadTimeline + DownloadStories = u.MediaDownloadStories DownloadHighlights = u.MediaDownloadHighlights DownloadChatMedia = u.MediaDownloadChatMedia MySettings = u.HOST.Source End Sub Friend Sub New(ByVal s As SiteSettings) + DownloadTimeline = s.DownloadTimeline.Value + DownloadStories = s.DownloadStories.Value DownloadHighlights = s.DownloadHighlights.Value DownloadChatMedia = s.DownloadChatMedia.Value MySettings = s diff --git a/SCrawler/API/PornHub/UserData.vb b/SCrawler/API/PornHub/UserData.vb index 078d4b3..f0e71ab 100644 --- a/SCrawler/API/PornHub/UserData.vb +++ b/SCrawler/API/PornHub/UserData.vb @@ -404,7 +404,9 @@ Namespace API.PornHub Dim r$ = Responser.GetResponse(URL) If Not r.IsEmptyString Then Dim l As List(Of UserVideo) = RegexFields(Of UserVideo)(r, {RegexUserVideos}, {6, 7, 3, 10}) - If l.ListExists And Not SiteMode = SiteModes.Playlists Then l = l.ListTake(3, l.Count).ToList + 'URGENT: PornHub: changed list trimming + 'If l.ListExists And Not SiteMode = SiteModes.Playlists Then l = l.ListTake(3, l.Count).ToList + If l.ListExists And Not SiteMode = SiteModes.Playlists Then l = l.ListTake(1, l.Count).ToList If l.ListExists Then If IsUser Then If Type = VideoTypes.Favorite Then diff --git a/SCrawler/Download/Feed/FeedMedia.vb b/SCrawler/Download/Feed/FeedMedia.vb index 39f737e..466a169 100644 --- a/SCrawler/Download/Feed/FeedMedia.vb +++ b/SCrawler/Download/Feed/FeedMedia.vb @@ -498,7 +498,7 @@ Namespace DownloadObjects If f.DialogResult = DialogResult.OK Then moveOptions = f.Result End Using If Not moveOptions.Destination.IsEmptyString Then - ff.Path = moveOptions.Destination.Path + ff.Path = moveOptions.DestinationTrue(Media).Path If isCopy Then result = File.Copy(ff) Else